From cda2fe68a3a549e2443ca7137b4df431af6f2ba7 Mon Sep 17 00:00:00 2001 From: Luke Whiting Date: Wed, 4 Dec 2024 10:55:13 +0000 Subject: [PATCH 001/204] [CI] DocsClientYamlTestSuiteIT test {yaml=reference/watcher/example-watches/example-watch-clusterstatus/line_137} failing - (#115809) (#117354) * Ignore system index access errors in YAML test index cleanup method * Remove test mute * Swap the logic back as it was right the first time * Resolve conflict with latest merge * Move warning handler into it's own method to reduce nesting --- muted-tests.yml | 3 -- .../test/rest/ESRestTestCase.java | 41 +++++++++---------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index b2f5b08319ff7..17a7b26d1c091 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -100,9 +100,6 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=transform/transforms_start_stop/Verify start transform reuses destination index} issue: https://github.com/elastic/elasticsearch/issues/115808 -- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT - method: test {yaml=reference/watcher/example-watches/example-watch-clusterstatus/line_137} - issue: https://github.com/elastic/elasticsearch/issues/115809 - class: org.elasticsearch.search.StressSearchServiceReaperIT method: testStressReaper issue: https://github.com/elastic/elasticsearch/issues/115816 diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index bdef0ba631b72..b4f4243fb90fd 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -26,6 +26,8 @@ import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; @@ -158,6 +160,8 @@ public abstract class ESRestTestCase extends ESTestCase { private static final Pattern SEMANTIC_VERSION_PATTERN = Pattern.compile("^(\\d+\\.\\d+\\.\\d+)\\D?.*"); + private static final Logger SUITE_LOGGER = LogManager.getLogger(ESRestTestCase.class); + /** * Convert the entity from a {@link Response} into a map of maps. * Consumes the underlying HttpEntity, releasing any resources it may be holding. @@ -1111,7 +1115,14 @@ protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOE } final Request deleteRequest = new Request("DELETE", Strings.collectionToCommaDelimitedString(indexPatterns)); deleteRequest.addParameter("expand_wildcards", "open,closed,hidden"); - deleteRequest.setOptions(deleteRequest.getOptions().toBuilder().setWarningsHandler(ignoreAsyncSearchWarning()).build()); + + // If system index warning, ignore but log + // See: https://github.com/elastic/elasticsearch/issues/117099 + // and: https://github.com/elastic/elasticsearch/issues/115809 + deleteRequest.setOptions( + RequestOptions.DEFAULT.toBuilder().setWarningsHandler(ESRestTestCase::ignoreSystemIndexAccessWarnings) + ); + final Response response = adminClient().performRequest(deleteRequest); try (InputStream is = response.getEntity().getContent()) { assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged")); @@ -1124,28 +1135,16 @@ protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOE } } - // Make warnings handler that ignores the .async-search warning since .async-search may randomly appear when async requests are slow - // See: https://github.com/elastic/elasticsearch/issues/117099 - protected static WarningsHandler ignoreAsyncSearchWarning() { - return new WarningsHandler() { - @Override - public boolean warningsShouldFailRequest(List warnings) { - if (warnings.isEmpty()) { - return false; - } - return warnings.equals( - List.of( - "this request accesses system indices: [.async-search], " - + "but in a future major version, direct access to system indices will be prevented by default" - ) - ) == false; + private static boolean ignoreSystemIndexAccessWarnings(List warnings) { + for (String warning : warnings) { + if (warning.startsWith("this request accesses system indices:")) { + SUITE_LOGGER.warn("Ignoring system index access warning during test cleanup: {}", warning); + } else { + return true; } + } - @Override - public String toString() { - return "ignore .async-search warning"; - } - }; + return false; } protected static void wipeDataStreams() throws IOException { From 4496aa0cbf5b23bc3146d4dd09d2b6194e3c9e5f Mon Sep 17 00:00:00 2001 From: Jakob Reiter Date: Wed, 4 Dec 2024 11:56:23 +0100 Subject: [PATCH 002/204] Update troubleshooting-unstable-cluster.asciidoc (#117887) Added missing word --- .../troubleshooting/troubleshooting-unstable-cluster.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc index cbb35f7731034..e47b85aa99547 100644 --- a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc +++ b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc @@ -126,7 +126,7 @@ repeatedly-dropped connections will severely affect its operation. The connections from the elected master node to every other node in the cluster are particularly important. The elected master never spontaneously closes its outbound connections to other nodes. Similarly, once an inbound connection is -fully established, a node never spontaneously it unless the node is shutting +fully established, a node never spontaneously closes it unless the node is shutting down. If you see a node unexpectedly leave the cluster with the `disconnected` From 032b42fcf7c398391d8197c5dcdbf7ee20628ce7 Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:17:13 +0100 Subject: [PATCH 003/204] Make TransportLocalClusterStateAction wait for cluster to unblock (#117230) This will make `TransportLocalClusterStateAction` wait for a new state that is not blocked. This means we need a timeout (again). For consistency's sake, we're reusing the REST param `master_timeout` for this timeout as well. The only class that was using `TransportLocalClusterStateAction` was `TransportGetAliasesAction`, so its request needed to accept a timeout again as well. --- docs/changelog/117230.yaml | 5 + docs/reference/cat/alias.asciidoc | 2 + docs/reference/indices/alias-exists.asciidoc | 2 + docs/reference/indices/get-alias.asciidoc | 2 + .../rest-api-spec/api/cat.aliases.json | 5 + .../api/indices.exists_alias.json | 5 + .../rest-api-spec/api/indices.get_alias.json | 5 + server/src/main/java/module-info.java | 1 + .../indices/alias/get/GetAliasesRequest.java | 21 ++- .../alias/get/TransportGetAliasesAction.java | 2 +- .../TransportLocalClusterStateAction.java | 64 ------- .../local/LocalClusterStateRequest.java | 43 +++++ .../TransportLocalClusterStateAction.java | 128 +++++++++++++ .../admin/indices/RestGetAliasesAction.java | 4 +- .../rest/action/cat/RestAliasAction.java | 9 +- ...TransportLocalClusterStateActionTests.java | 177 ++++++++++++++++++ 16 files changed, 396 insertions(+), 79 deletions(-) create mode 100644 docs/changelog/117230.yaml delete mode 100644 server/src/main/java/org/elasticsearch/action/support/TransportLocalClusterStateAction.java create mode 100644 server/src/main/java/org/elasticsearch/action/support/local/LocalClusterStateRequest.java create mode 100644 server/src/main/java/org/elasticsearch/action/support/local/TransportLocalClusterStateAction.java create mode 100644 server/src/test/java/org/elasticsearch/action/support/local/TransportLocalClusterStateActionTests.java diff --git a/docs/changelog/117230.yaml b/docs/changelog/117230.yaml new file mode 100644 index 0000000000000..001dcef2fe3b1 --- /dev/null +++ b/docs/changelog/117230.yaml @@ -0,0 +1,5 @@ +pr: 117230 +summary: Make various alias retrieval APIs wait for cluster to unblock +area: Distributed +type: enhancement +issues: [] diff --git a/docs/reference/cat/alias.asciidoc b/docs/reference/cat/alias.asciidoc index 41ac279d3b2f5..aab0c9df25ed4 100644 --- a/docs/reference/cat/alias.asciidoc +++ b/docs/reference/cat/alias.asciidoc @@ -51,6 +51,8 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] + [[cat-alias-api-example]] ==== {api-examples-title} diff --git a/docs/reference/indices/alias-exists.asciidoc b/docs/reference/indices/alias-exists.asciidoc index d7b3454dcff56..a514d36a1bfef 100644 --- a/docs/reference/indices/alias-exists.asciidoc +++ b/docs/reference/indices/alias-exists.asciidoc @@ -52,6 +52,8 @@ Defaults to `all`. (Optional, Boolean) If `false`, requests that include a missing data stream or index in the `` return an error. Defaults to `false`. +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] + [[alias-exists-api-response-codes]] ==== {api-response-codes-title} diff --git a/docs/reference/indices/get-alias.asciidoc b/docs/reference/indices/get-alias.asciidoc index 41d62fb70e01b..d4c5b92116949 100644 --- a/docs/reference/indices/get-alias.asciidoc +++ b/docs/reference/indices/get-alias.asciidoc @@ -58,3 +58,5 @@ Defaults to `all`. `ignore_unavailable`:: (Optional, Boolean) If `false`, requests that include a missing data stream or index in the `` return an error. Defaults to `false`. + +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json index d3856b455efd1..9f97fe6280dc0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json @@ -65,6 +65,11 @@ ], "default": "all", "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both." + }, + "master_timeout":{ + "type":"time", + "description":"Timeout for waiting for new cluster state in case it is blocked", + "default":"30s" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json index 7d7a9c96c6419..7c855335efd00 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json @@ -61,6 +61,11 @@ ], "default":"all", "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both." + }, + "master_timeout":{ + "type":"time", + "description":"Timeout for waiting for new cluster state in case it is blocked", + "default":"30s" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json index dc02a65adb068..a360582a44a04 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json @@ -79,6 +79,11 @@ ], "default": "all", "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both." + }, + "master_timeout":{ + "type":"time", + "description":"Timeout for waiting for new cluster state in case it is blocked", + "default":"30s" } } } diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 5acc202ebb294..331a2bc0dddac 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -147,6 +147,7 @@ exports org.elasticsearch.action.support.master; exports org.elasticsearch.action.support.master.info; exports org.elasticsearch.action.support.nodes; + exports org.elasticsearch.action.support.local; exports org.elasticsearch.action.support.replication; exports org.elasticsearch.action.support.single.instance; exports org.elasticsearch.action.support.single.shard; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java index c28108815ed03..af95787a5fcca 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java @@ -8,21 +8,20 @@ */ package org.elasticsearch.action.admin.indices.alias.get; -import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; +import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import java.io.IOException; import java.util.Map; -public class GetAliasesRequest extends ActionRequest implements AliasesRequest { +public class GetAliasesRequest extends LocalClusterStateRequest implements AliasesRequest { public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandHidden(); @@ -31,18 +30,20 @@ public class GetAliasesRequest extends ActionRequest implements AliasesRequest { private String[] indices = Strings.EMPTY_ARRAY; private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; + @Deprecated public GetAliasesRequest(String... aliases) { - this.aliases = aliases; - this.originalAliases = aliases; + this(MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, aliases); } + @Deprecated public GetAliasesRequest() { this(Strings.EMPTY_ARRAY); } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); + public GetAliasesRequest(TimeValue masterTimeout, String... aliases) { + super(masterTimeout); + this.aliases = aliases; + this.originalAliases = aliases; } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index 67f1a627ae77f..9e0014b0f512b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -10,7 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.TransportLocalClusterStateAction; +import org.elasticsearch.action.support.local.TransportLocalClusterStateAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/elasticsearch/action/support/TransportLocalClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/support/TransportLocalClusterStateAction.java deleted file mode 100644 index d48dc74295bcb..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/support/TransportLocalClusterStateAction.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.support; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.ActionRunnable; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.tasks.TaskManager; - -import java.util.concurrent.Executor; - -/** - * Analogue of {@link org.elasticsearch.action.support.master.TransportMasterNodeReadAction} except that it runs on the local node rather - * than delegating to the master. - */ -public abstract class TransportLocalClusterStateAction extends - TransportAction { - - protected final ClusterService clusterService; - protected final Executor executor; - - protected TransportLocalClusterStateAction( - String actionName, - ActionFilters actionFilters, - TaskManager taskManager, - ClusterService clusterService, - Executor executor - ) { - // TODO replace DIRECT_EXECUTOR_SERVICE when removing workaround for https://github.com/elastic/elasticsearch/issues/97916 - super(actionName, actionFilters, taskManager, EsExecutors.DIRECT_EXECUTOR_SERVICE); - this.clusterService = clusterService; - this.executor = executor; - } - - protected abstract ClusterBlockException checkBlock(Request request, ClusterState state); - - @Override - protected final void doExecute(Task task, Request request, ActionListener listener) { - final var state = clusterService.state(); - final var clusterBlockException = checkBlock(request, state); - if (clusterBlockException != null) { - throw clusterBlockException; - } - - // Workaround for https://github.com/elastic/elasticsearch/issues/97916 - TODO remove this when we can - executor.execute(ActionRunnable.wrap(listener, l -> localClusterStateOperation(task, request, state, l))); - } - - protected abstract void localClusterStateOperation(Task task, Request request, ClusterState state, ActionListener listener) - throws Exception; -} diff --git a/server/src/main/java/org/elasticsearch/action/support/local/LocalClusterStateRequest.java b/server/src/main/java/org/elasticsearch/action/support/local/LocalClusterStateRequest.java new file mode 100644 index 0000000000000..dfbcb21c2a959 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/support/local/LocalClusterStateRequest.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.support.local; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; + +import java.io.IOException; +import java.util.Objects; + +/** + * A base request for actions that are executed locally on the node that receives the request. + */ +public abstract class LocalClusterStateRequest extends ActionRequest { + + /** + * The timeout for waiting until the cluster is unblocked. + * We use the name masterTimeout to be consistent with the master node actions. + */ + private final TimeValue masterTimeout; + + protected LocalClusterStateRequest(TimeValue masterTimeout) { + this.masterTimeout = Objects.requireNonNull(masterTimeout); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + TransportAction.localOnly(); + } + + public TimeValue masterTimeout() { + return masterTimeout; + } +} diff --git a/server/src/main/java/org/elasticsearch/action/support/local/TransportLocalClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/support/local/TransportLocalClusterStateAction.java new file mode 100644 index 0000000000000..66f94050c9826 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/support/local/TransportLocalClusterStateAction.java @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.support.local; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateObserver; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.node.NodeClosedException; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskManager; + +import java.util.concurrent.Executor; + +import static org.elasticsearch.common.Strings.format; + +/** + * Analogue of {@link org.elasticsearch.action.support.master.TransportMasterNodeReadAction} except that it runs on the local node rather + * than delegating to the master. + */ +public abstract class TransportLocalClusterStateAction extends + TransportAction { + + private static final Logger logger = LogManager.getLogger(TransportLocalClusterStateAction.class); + + protected final ClusterService clusterService; + protected final Executor executor; + + protected TransportLocalClusterStateAction( + String actionName, + ActionFilters actionFilters, + TaskManager taskManager, + ClusterService clusterService, + Executor executor + ) { + // TODO replace DIRECT_EXECUTOR_SERVICE when removing workaround for https://github.com/elastic/elasticsearch/issues/97916 + super(actionName, actionFilters, taskManager, EsExecutors.DIRECT_EXECUTOR_SERVICE); + this.clusterService = clusterService; + this.executor = executor; + } + + protected abstract ClusterBlockException checkBlock(Request request, ClusterState state); + + protected abstract void localClusterStateOperation(Task task, Request request, ClusterState state, ActionListener listener) + throws Exception; + + @Override + protected final void doExecute(Task task, Request request, ActionListener listener) { + final var state = clusterService.state(); + final var clusterBlockException = checkBlock(request, state); + if (clusterBlockException != null) { + if (clusterBlockException.retryable() == false) { + listener.onFailure(clusterBlockException); + } else { + waitForClusterUnblock(task, request, listener, state, clusterBlockException); + } + } else { + innerDoExecute(task, request, listener, state); + } + } + + private void innerDoExecute(Task task, Request request, ActionListener listener, ClusterState state) { + if (task instanceof CancellableTask cancellableTask && cancellableTask.notifyIfCancelled(listener)) { + return; + } + // Workaround for https://github.com/elastic/elasticsearch/issues/97916 - TODO remove this when we can + executor.execute(ActionRunnable.wrap(listener, l -> localClusterStateOperation(task, request, state, l))); + } + + private void waitForClusterUnblock( + Task task, + Request request, + ActionListener listener, + ClusterState initialState, + ClusterBlockException exception + ) { + var observer = new ClusterStateObserver( + initialState, + clusterService, + request.masterTimeout(), + logger, + clusterService.threadPool().getThreadContext() + ); + observer.waitForNextChange(new ClusterStateObserver.Listener() { + @Override + public void onNewClusterState(ClusterState state) { + logger.trace("retrying with cluster state version [{}]", state.version()); + innerDoExecute(task, request, listener, state); + } + + @Override + public void onClusterServiceClose() { + listener.onFailure(new NodeClosedException(clusterService.localNode())); + } + + @Override + public void onTimeout(TimeValue timeout) { + logger.debug( + () -> format("timed out while waiting for cluster to unblock in [%s] (timeout [%s])", actionName, timeout), + exception + ); + listener.onFailure(new ElasticsearchTimeoutException("timed out while waiting for cluster to unblock", exception)); + } + }, clusterState -> isTaskCancelled(task) || checkBlock(request, clusterState) == null); + } + + private boolean isTaskCancelled(Task task) { + return task instanceof CancellableTask cancellableTask && cancellableTask.isCancelled(); + } +} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java index dfe501f29ce2e..841d7979862dc 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestBuilderListener; @@ -207,7 +208,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final boolean namesProvided = request.hasParam("name"); final String[] aliases = request.paramAsStringArrayOrEmptyIfAll("name"); - final GetAliasesRequest getAliasesRequest = new GetAliasesRequest(aliases); + final var masterNodeTimeout = RestUtils.getMasterNodeTimeout(request); + final GetAliasesRequest getAliasesRequest = new GetAliasesRequest(masterNodeTimeout, aliases); final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); getAliasesRequest.indices(indices); getAliasesRequest.indicesOptions(IndicesOptions.fromRequest(request, getAliasesRequest.indicesOptions())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java index 6aa0b1c865682..cfb714c354f02 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.Table; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestCancellableNodeClient; @@ -47,9 +48,11 @@ public boolean allowSystemIndexAccessByDefault() { @Override protected RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { - final GetAliasesRequest getAliasesRequest = request.hasParam("alias") - ? new GetAliasesRequest(Strings.commaDelimitedListToStringArray(request.param("alias"))) - : new GetAliasesRequest(); + final var masterNodeTimeout = RestUtils.getMasterNodeTimeout(request); + final GetAliasesRequest getAliasesRequest = new GetAliasesRequest( + masterNodeTimeout, + Strings.commaDelimitedListToStringArray(request.param("alias")) + ); getAliasesRequest.indicesOptions(IndicesOptions.fromRequest(request, getAliasesRequest.indicesOptions())); return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).admin() .indices() diff --git a/server/src/test/java/org/elasticsearch/action/support/local/TransportLocalClusterStateActionTests.java b/server/src/test/java/org/elasticsearch/action/support/local/TransportLocalClusterStateActionTests.java new file mode 100644 index 0000000000000..ea71cefc6608c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/support/local/TransportLocalClusterStateActionTests.java @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.support.local; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ActionTestUtils; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlock; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelHelper; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; +import static org.elasticsearch.test.ClusterServiceUtils.setState; + +public class TransportLocalClusterStateActionTests extends ESTestCase { + + private static ThreadPool threadPool; + + private ClusterService clusterService; + private TaskManager taskManager; + + @BeforeClass + public static void beforeClass() { + threadPool = new TestThreadPool(getTestClass().getName()); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + clusterService = createClusterService(threadPool); + taskManager = new TaskManager(clusterService.getSettings(), threadPool, Set.of()); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + clusterService.close(); + } + + @AfterClass + public static void afterClass() { + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + threadPool = null; + } + + public void testNoBlock() throws ExecutionException, InterruptedException { + var request = new Request(); + PlainActionFuture listener = new PlainActionFuture<>(); + ActionTestUtils.execute(new Action(taskManager, clusterService), null, request, listener); + assertTrue(listener.isDone()); + listener.get(); + } + + public void testRetryAfterBlock() throws ExecutionException, InterruptedException { + var request = new Request(); + ClusterBlock block = new ClusterBlock(randomInt(), "", true, true, false, randomFrom(RestStatus.values()), ClusterBlockLevel.ALL); + var state = ClusterState.builder(clusterService.state()).blocks(ClusterBlocks.builder().addGlobalBlock(block)).build(); + setState(clusterService, state); + + PlainActionFuture listener = new PlainActionFuture<>(); + ActionTestUtils.execute(new Action(taskManager, clusterService), null, request, listener); + + assertFalse(listener.isDone()); + setState(clusterService, ClusterState.builder(state).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build()); + assertTrue(listener.isDone()); + listener.get(); + } + + public void testNonRetryableBlock() { + var request = new Request(); + ClusterBlock block = new ClusterBlock(randomInt(), "", false, true, false, randomFrom(RestStatus.values()), ClusterBlockLevel.ALL); + var state = ClusterState.builder(clusterService.state()).blocks(ClusterBlocks.builder().addGlobalBlock(block)).build(); + setState(clusterService, state); + + PlainActionFuture listener = new PlainActionFuture<>(); + ActionTestUtils.execute(new Action(taskManager, clusterService), null, request, listener); + + assertTrue(listener.isDone()); + var exception = assertThrows(ExecutionException.class, listener::get); + assertTrue(exception.getCause() instanceof ClusterBlockException); + } + + public void testTaskCancelledAfterBlock() { + var request = new Request(); + ClusterBlock block = new ClusterBlock(randomInt(), "", true, true, false, randomFrom(RestStatus.values()), ClusterBlockLevel.ALL); + var state = ClusterState.builder(clusterService.state()).blocks(ClusterBlocks.builder().addGlobalBlock(block)).build(); + setState(clusterService, state); + + CancellableTask task = new CancellableTask(randomLong(), "test", Action.ACTION_NAME, "", TaskId.EMPTY_TASK_ID, Map.of()); + PlainActionFuture listener = new PlainActionFuture<>(); + ActionTestUtils.execute(new Action(taskManager, clusterService), task, request, listener); + + TaskCancelHelper.cancel(task, "test"); + assertFalse(listener.isDone()); + setState(clusterService, ClusterState.builder(state).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build()); + assertTrue(listener.isDone()); + var exception = assertThrows(ExecutionException.class, listener::get); + assertTrue(exception.getCause() instanceof TaskCancelledException); + } + + private static class Request extends LocalClusterStateRequest { + + protected Request() { + super(TEST_REQUEST_TIMEOUT); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + private static class Response extends ActionResponse { + + @Override + public void writeTo(StreamOutput out) throws IOException { + TransportAction.localOnly(); + } + } + + private static class Action extends TransportLocalClusterStateAction { + static final String ACTION_NAME = "internal:testAction"; + + Action(TaskManager taskManager, ClusterService clusterService) { + super(ACTION_NAME, new ActionFilters(Set.of()), taskManager, clusterService, EsExecutors.DIRECT_EXECUTOR_SERVICE); + } + + @Override + protected void localClusterStateOperation(Task task, Request request, ClusterState state, ActionListener listener) + throws Exception { + listener.onResponse(new Response()); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + Set blocks = state.blocks().global(); + return blocks.isEmpty() ? null : new ClusterBlockException(blocks); + } + } +} From 9d9eed5bad0ecbaeaa6397b992eff46c5f653a7d Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 4 Dec 2024 22:20:04 +1100 Subject: [PATCH 004/204] Assert for empty routing table when hiding state (#117870) Routing table should always be empty when hiding cluster state for not-recovered block. This PR adds an explicit assertion for it. --- .../java/org/elasticsearch/gateway/ClusterStateUpdaters.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java b/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java index 8d8e661b3a010..9c16bcd9d2cc2 100644 --- a/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java +++ b/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java @@ -143,6 +143,7 @@ public static ClusterState hideStateIfNotRecovered(ClusterState state) { .coordinationMetadata(state.metadata().coordinationMetadata()) .build(); + assert state.routingTable().indicesRouting().isEmpty() : "routing table is not empty: " + state.routingTable().indicesRouting(); return ClusterState.builder(state).metadata(metadata).blocks(blocks.build()).build(); } return state; From c7e985c3b61ef1aece751e674f5431fb06174935 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 4 Dec 2024 12:20:47 +0100 Subject: [PATCH 005/204] Support ST_ENVELOPE and related ST_XMIN, etc. (#116964) Support ST_ENVELOPE and related ST_XMIN, etc. Based on the PostGIS equivalents: https://postgis.net/docs/ST_Envelope.html https://postgis.net/docs/ST_XMin.html https://postgis.net/docs/ST_XMax.html https://postgis.net/docs/ST_YMin.html https://postgis.net/docs/ST_YMax.html --- docs/changelog/116964.yaml | 6 + .../description/st_envelope.asciidoc | 5 + .../functions/description/st_xmax.asciidoc | 5 + .../functions/description/st_xmin.asciidoc | 5 + .../functions/description/st_ymax.asciidoc | 5 + .../functions/description/st_ymin.asciidoc | 5 + .../functions/examples/st_envelope.asciidoc | 13 + .../esql/functions/examples/st_xmax.asciidoc | 13 + .../esql/functions/examples/st_xmin.asciidoc | 13 + .../esql/functions/examples/st_ymax.asciidoc | 13 + .../esql/functions/examples/st_ymin.asciidoc | 13 + .../kibana/definition/st_envelope.json | 61 +++ .../functions/kibana/definition/st_xmax.json | 61 +++ .../functions/kibana/definition/st_xmin.json | 61 +++ .../functions/kibana/definition/st_ymax.json | 61 +++ .../functions/kibana/definition/st_ymin.json | 61 +++ .../esql/functions/kibana/docs/st_envelope.md | 13 + .../esql/functions/kibana/docs/st_xmax.md | 15 + .../esql/functions/kibana/docs/st_xmin.md | 15 + .../esql/functions/kibana/docs/st_ymax.md | 15 + .../esql/functions/kibana/docs/st_ymin.md | 15 + .../functions/layout/st_envelope.asciidoc | 15 + .../esql/functions/layout/st_xmax.asciidoc | 15 + .../esql/functions/layout/st_xmin.asciidoc | 15 + .../esql/functions/layout/st_ymax.asciidoc | 15 + .../esql/functions/layout/st_ymin.asciidoc | 15 + .../functions/parameters/st_envelope.asciidoc | 6 + .../functions/parameters/st_xmax.asciidoc | 6 + .../functions/parameters/st_xmin.asciidoc | 6 + .../functions/parameters/st_ymax.asciidoc | 6 + .../functions/parameters/st_ymin.asciidoc | 6 + .../esql/functions/signature/st_envelope.svg | 1 + .../esql/functions/signature/st_xmax.svg | 1 + .../esql/functions/signature/st_xmin.svg | 1 + .../esql/functions/signature/st_ymax.svg | 1 + .../esql/functions/signature/st_ymin.svg | 1 + .../esql/functions/spatial-functions.asciidoc | 10 + .../esql/functions/types/st_envelope.asciidoc | 12 + .../esql/functions/types/st_xmax.asciidoc | 12 + .../esql/functions/types/st_xmin.asciidoc | 12 + .../esql/functions/types/st_ymax.asciidoc | 12 + .../esql/functions/types/st_ymin.asciidoc | 12 + .../utils/SpatialEnvelopeVisitor.java | 356 ++++++++++++++++++ .../utils/SpatialEnvelopeVisitorTests.java | 194 ++++++++++ .../main/resources/spatial_shapes.csv-spec | 41 ++ .../spatial/StEnvelopeFromWKBEvaluator.java | 126 +++++++ .../StEnvelopeFromWKBGeoEvaluator.java | 126 +++++++ .../spatial/StXMaxFromWKBEvaluator.java | 127 +++++++ .../spatial/StXMaxFromWKBGeoEvaluator.java | 127 +++++++ .../spatial/StXMinFromWKBEvaluator.java | 127 +++++++ .../spatial/StXMinFromWKBGeoEvaluator.java | 127 +++++++ .../spatial/StYMaxFromWKBEvaluator.java | 127 +++++++ .../spatial/StYMaxFromWKBGeoEvaluator.java | 127 +++++++ .../spatial/StYMinFromWKBEvaluator.java | 127 +++++++ .../spatial/StYMinFromWKBGeoEvaluator.java | 127 +++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../esql/expression/ExpressionWritables.java | 10 + .../function/EsqlFunctionRegistry.java | 10 + .../function/scalar/spatial/StEnvelope.java | 138 +++++++ .../function/scalar/spatial/StXMax.java | 123 ++++++ .../function/scalar/spatial/StXMin.java | 123 ++++++ .../function/scalar/spatial/StYMax.java | 123 ++++++ .../function/scalar/spatial/StYMin.java | 123 ++++++ .../scalar/spatial/StEnvelopeTests.java | 88 +++++ .../function/scalar/spatial/StXMaxTests.java | 75 ++++ .../function/scalar/spatial/StXMinTests.java | 75 ++++ .../function/scalar/spatial/StYMaxTests.java | 75 ++++ .../function/scalar/spatial/StYMinTests.java | 75 ++++ .../rest-api-spec/test/esql/60_usage.yml | 4 +- 69 files changed, 3558 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/116964.yaml create mode 100644 docs/reference/esql/functions/description/st_envelope.asciidoc create mode 100644 docs/reference/esql/functions/description/st_xmax.asciidoc create mode 100644 docs/reference/esql/functions/description/st_xmin.asciidoc create mode 100644 docs/reference/esql/functions/description/st_ymax.asciidoc create mode 100644 docs/reference/esql/functions/description/st_ymin.asciidoc create mode 100644 docs/reference/esql/functions/examples/st_envelope.asciidoc create mode 100644 docs/reference/esql/functions/examples/st_xmax.asciidoc create mode 100644 docs/reference/esql/functions/examples/st_xmin.asciidoc create mode 100644 docs/reference/esql/functions/examples/st_ymax.asciidoc create mode 100644 docs/reference/esql/functions/examples/st_ymin.asciidoc create mode 100644 docs/reference/esql/functions/kibana/definition/st_envelope.json create mode 100644 docs/reference/esql/functions/kibana/definition/st_xmax.json create mode 100644 docs/reference/esql/functions/kibana/definition/st_xmin.json create mode 100644 docs/reference/esql/functions/kibana/definition/st_ymax.json create mode 100644 docs/reference/esql/functions/kibana/definition/st_ymin.json create mode 100644 docs/reference/esql/functions/kibana/docs/st_envelope.md create mode 100644 docs/reference/esql/functions/kibana/docs/st_xmax.md create mode 100644 docs/reference/esql/functions/kibana/docs/st_xmin.md create mode 100644 docs/reference/esql/functions/kibana/docs/st_ymax.md create mode 100644 docs/reference/esql/functions/kibana/docs/st_ymin.md create mode 100644 docs/reference/esql/functions/layout/st_envelope.asciidoc create mode 100644 docs/reference/esql/functions/layout/st_xmax.asciidoc create mode 100644 docs/reference/esql/functions/layout/st_xmin.asciidoc create mode 100644 docs/reference/esql/functions/layout/st_ymax.asciidoc create mode 100644 docs/reference/esql/functions/layout/st_ymin.asciidoc create mode 100644 docs/reference/esql/functions/parameters/st_envelope.asciidoc create mode 100644 docs/reference/esql/functions/parameters/st_xmax.asciidoc create mode 100644 docs/reference/esql/functions/parameters/st_xmin.asciidoc create mode 100644 docs/reference/esql/functions/parameters/st_ymax.asciidoc create mode 100644 docs/reference/esql/functions/parameters/st_ymin.asciidoc create mode 100644 docs/reference/esql/functions/signature/st_envelope.svg create mode 100644 docs/reference/esql/functions/signature/st_xmax.svg create mode 100644 docs/reference/esql/functions/signature/st_xmin.svg create mode 100644 docs/reference/esql/functions/signature/st_ymax.svg create mode 100644 docs/reference/esql/functions/signature/st_ymin.svg create mode 100644 docs/reference/esql/functions/types/st_envelope.asciidoc create mode 100644 docs/reference/esql/functions/types/st_xmax.asciidoc create mode 100644 docs/reference/esql/functions/types/st_xmin.asciidoc create mode 100644 docs/reference/esql/functions/types/st_ymax.asciidoc create mode 100644 docs/reference/esql/functions/types/st_ymin.asciidoc create mode 100644 libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java create mode 100644 libs/geo/src/test/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitorTests.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeFromWKBEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeFromWKBGeoEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBGeoEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBGeoEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBGeoEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBGeoEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelope.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMax.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMin.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMax.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMin.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinTests.java diff --git a/docs/changelog/116964.yaml b/docs/changelog/116964.yaml new file mode 100644 index 0000000000000..2e3ecd06fa098 --- /dev/null +++ b/docs/changelog/116964.yaml @@ -0,0 +1,6 @@ +pr: 116964 +summary: "Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions" +area: ES|QL +type: feature +issues: + - 104875 diff --git a/docs/reference/esql/functions/description/st_envelope.asciidoc b/docs/reference/esql/functions/description/st_envelope.asciidoc new file mode 100644 index 0000000000000..6b7cf8d97538a --- /dev/null +++ b/docs/reference/esql/functions/description/st_envelope.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Determines the minimum bounding box of the supplied geometry. diff --git a/docs/reference/esql/functions/description/st_xmax.asciidoc b/docs/reference/esql/functions/description/st_xmax.asciidoc new file mode 100644 index 0000000000000..f33ec590bf2d4 --- /dev/null +++ b/docs/reference/esql/functions/description/st_xmax.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Extracts the maximum value of the `x` coordinates from the supplied geometry. If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `longitude` value. diff --git a/docs/reference/esql/functions/description/st_xmin.asciidoc b/docs/reference/esql/functions/description/st_xmin.asciidoc new file mode 100644 index 0000000000000..b06cbfacde7bf --- /dev/null +++ b/docs/reference/esql/functions/description/st_xmin.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Extracts the minimum value of the `x` coordinates from the supplied geometry. If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `longitude` value. diff --git a/docs/reference/esql/functions/description/st_ymax.asciidoc b/docs/reference/esql/functions/description/st_ymax.asciidoc new file mode 100644 index 0000000000000..f9475dd967562 --- /dev/null +++ b/docs/reference/esql/functions/description/st_ymax.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Extracts the maximum value of the `y` coordinates from the supplied geometry. If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `latitude` value. diff --git a/docs/reference/esql/functions/description/st_ymin.asciidoc b/docs/reference/esql/functions/description/st_ymin.asciidoc new file mode 100644 index 0000000000000..7228c63a16030 --- /dev/null +++ b/docs/reference/esql/functions/description/st_ymin.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Extracts the minimum value of the `y` coordinates from the supplied geometry. If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `latitude` value. diff --git a/docs/reference/esql/functions/examples/st_envelope.asciidoc b/docs/reference/esql/functions/examples/st_envelope.asciidoc new file mode 100644 index 0000000000000..df8c0ad5607fa --- /dev/null +++ b/docs/reference/esql/functions/examples/st_envelope.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_envelope] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_envelope-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_xmax.asciidoc b/docs/reference/esql/functions/examples/st_xmax.asciidoc new file mode 100644 index 0000000000000..5bba1761cf29c --- /dev/null +++ b/docs/reference/esql/functions/examples/st_xmax.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_xmin.asciidoc b/docs/reference/esql/functions/examples/st_xmin.asciidoc new file mode 100644 index 0000000000000..5bba1761cf29c --- /dev/null +++ b/docs/reference/esql/functions/examples/st_xmin.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_ymax.asciidoc b/docs/reference/esql/functions/examples/st_ymax.asciidoc new file mode 100644 index 0000000000000..5bba1761cf29c --- /dev/null +++ b/docs/reference/esql/functions/examples/st_ymax.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_ymin.asciidoc b/docs/reference/esql/functions/examples/st_ymin.asciidoc new file mode 100644 index 0000000000000..5bba1761cf29c --- /dev/null +++ b/docs/reference/esql/functions/examples/st_ymin.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/st_envelope.json b/docs/reference/esql/functions/kibana/definition/st_envelope.json new file mode 100644 index 0000000000000..6c00dda265ac7 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_envelope.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_envelope", + "description" : "Determines the minimum bounding box of the supplied geometry.", + "signatures" : [ + { + "params" : [ + { + "name" : "geometry", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "cartesian_shape" + }, + { + "params" : [ + { + "name" : "geometry", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "cartesian_shape" + }, + { + "params" : [ + { + "name" : "geometry", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "geo_shape" + }, + { + "params" : [ + { + "name" : "geometry", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "geo_shape" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| KEEP abbrev, airport, envelope" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_xmax.json b/docs/reference/esql/functions/kibana/definition/st_xmax.json new file mode 100644 index 0000000000000..7be22617c0992 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_xmax.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_xmax", + "description" : "Extracts the maximum value of the `x` coordinates from the supplied geometry.\nIf the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `longitude` value.", + "signatures" : [ + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope)\n| KEEP abbrev, airport, xmin, xmax, ymin, ymax" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_xmin.json b/docs/reference/esql/functions/kibana/definition/st_xmin.json new file mode 100644 index 0000000000000..8052fdb861cea --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_xmin.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_xmin", + "description" : "Extracts the minimum value of the `x` coordinates from the supplied geometry.\nIf the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `longitude` value.", + "signatures" : [ + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope)\n| KEEP abbrev, airport, xmin, xmax, ymin, ymax" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_ymax.json b/docs/reference/esql/functions/kibana/definition/st_ymax.json new file mode 100644 index 0000000000000..1a53f7388ea56 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_ymax.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_ymax", + "description" : "Extracts the maximum value of the `y` coordinates from the supplied geometry.\nIf the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `latitude` value.", + "signatures" : [ + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope)\n| KEEP abbrev, airport, xmin, xmax, ymin, ymax" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_ymin.json b/docs/reference/esql/functions/kibana/definition/st_ymin.json new file mode 100644 index 0000000000000..e11722a8f9c07 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_ymin.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_ymin", + "description" : "Extracts the minimum value of the `y` coordinates from the supplied geometry.\nIf the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `latitude` value.", + "signatures" : [ + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope)\n| KEEP abbrev, airport, xmin, xmax, ymin, ymax" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/docs/st_envelope.md b/docs/reference/esql/functions/kibana/docs/st_envelope.md new file mode 100644 index 0000000000000..5f4c3e4809a82 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_envelope.md @@ -0,0 +1,13 @@ + + +### ST_ENVELOPE +Determines the minimum bounding box of the supplied geometry. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| KEEP abbrev, airport, envelope +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_xmax.md b/docs/reference/esql/functions/kibana/docs/st_xmax.md new file mode 100644 index 0000000000000..bbde89df76fd0 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_xmax.md @@ -0,0 +1,15 @@ + + +### ST_XMAX +Extracts the maximum value of the `x` coordinates from the supplied geometry. +If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `longitude` value. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) +| KEEP abbrev, airport, xmin, xmax, ymin, ymax +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_xmin.md b/docs/reference/esql/functions/kibana/docs/st_xmin.md new file mode 100644 index 0000000000000..1a6cee7dcfd62 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_xmin.md @@ -0,0 +1,15 @@ + + +### ST_XMIN +Extracts the minimum value of the `x` coordinates from the supplied geometry. +If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `longitude` value. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) +| KEEP abbrev, airport, xmin, xmax, ymin, ymax +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_ymax.md b/docs/reference/esql/functions/kibana/docs/st_ymax.md new file mode 100644 index 0000000000000..61c9b6c288ca5 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_ymax.md @@ -0,0 +1,15 @@ + + +### ST_YMAX +Extracts the maximum value of the `y` coordinates from the supplied geometry. +If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `latitude` value. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) +| KEEP abbrev, airport, xmin, xmax, ymin, ymax +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_ymin.md b/docs/reference/esql/functions/kibana/docs/st_ymin.md new file mode 100644 index 0000000000000..f5817f10f20a5 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_ymin.md @@ -0,0 +1,15 @@ + + +### ST_YMIN +Extracts the minimum value of the `y` coordinates from the supplied geometry. +If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `latitude` value. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) +| KEEP abbrev, airport, xmin, xmax, ymin, ymax +``` diff --git a/docs/reference/esql/functions/layout/st_envelope.asciidoc b/docs/reference/esql/functions/layout/st_envelope.asciidoc new file mode 100644 index 0000000000000..a20d4275e0c9f --- /dev/null +++ b/docs/reference/esql/functions/layout/st_envelope.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_envelope]] +=== `ST_ENVELOPE` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_envelope.svg[Embedded,opts=inline] + +include::../parameters/st_envelope.asciidoc[] +include::../description/st_envelope.asciidoc[] +include::../types/st_envelope.asciidoc[] +include::../examples/st_envelope.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_xmax.asciidoc b/docs/reference/esql/functions/layout/st_xmax.asciidoc new file mode 100644 index 0000000000000..b0c5e7695521e --- /dev/null +++ b/docs/reference/esql/functions/layout/st_xmax.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_xmax]] +=== `ST_XMAX` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_xmax.svg[Embedded,opts=inline] + +include::../parameters/st_xmax.asciidoc[] +include::../description/st_xmax.asciidoc[] +include::../types/st_xmax.asciidoc[] +include::../examples/st_xmax.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_xmin.asciidoc b/docs/reference/esql/functions/layout/st_xmin.asciidoc new file mode 100644 index 0000000000000..55fbad88c4cf0 --- /dev/null +++ b/docs/reference/esql/functions/layout/st_xmin.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_xmin]] +=== `ST_XMIN` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_xmin.svg[Embedded,opts=inline] + +include::../parameters/st_xmin.asciidoc[] +include::../description/st_xmin.asciidoc[] +include::../types/st_xmin.asciidoc[] +include::../examples/st_xmin.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_ymax.asciidoc b/docs/reference/esql/functions/layout/st_ymax.asciidoc new file mode 100644 index 0000000000000..e1022de4ba664 --- /dev/null +++ b/docs/reference/esql/functions/layout/st_ymax.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_ymax]] +=== `ST_YMAX` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_ymax.svg[Embedded,opts=inline] + +include::../parameters/st_ymax.asciidoc[] +include::../description/st_ymax.asciidoc[] +include::../types/st_ymax.asciidoc[] +include::../examples/st_ymax.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_ymin.asciidoc b/docs/reference/esql/functions/layout/st_ymin.asciidoc new file mode 100644 index 0000000000000..65511e1925e27 --- /dev/null +++ b/docs/reference/esql/functions/layout/st_ymin.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_ymin]] +=== `ST_YMIN` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_ymin.svg[Embedded,opts=inline] + +include::../parameters/st_ymin.asciidoc[] +include::../description/st_ymin.asciidoc[] +include::../types/st_ymin.asciidoc[] +include::../examples/st_ymin.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/st_envelope.asciidoc b/docs/reference/esql/functions/parameters/st_envelope.asciidoc new file mode 100644 index 0000000000000..a31c6a85de367 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_envelope.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`geometry`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_xmax.asciidoc b/docs/reference/esql/functions/parameters/st_xmax.asciidoc new file mode 100644 index 0000000000000..788f3485af297 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_xmax.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`point`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_xmin.asciidoc b/docs/reference/esql/functions/parameters/st_xmin.asciidoc new file mode 100644 index 0000000000000..788f3485af297 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_xmin.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`point`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_ymax.asciidoc b/docs/reference/esql/functions/parameters/st_ymax.asciidoc new file mode 100644 index 0000000000000..788f3485af297 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_ymax.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`point`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_ymin.asciidoc b/docs/reference/esql/functions/parameters/st_ymin.asciidoc new file mode 100644 index 0000000000000..788f3485af297 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_ymin.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`point`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/signature/st_envelope.svg b/docs/reference/esql/functions/signature/st_envelope.svg new file mode 100644 index 0000000000000..885a60e6fd86f --- /dev/null +++ b/docs/reference/esql/functions/signature/st_envelope.svg @@ -0,0 +1 @@ +ST_ENVELOPE(geometry) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_xmax.svg b/docs/reference/esql/functions/signature/st_xmax.svg new file mode 100644 index 0000000000000..348d5a7f72763 --- /dev/null +++ b/docs/reference/esql/functions/signature/st_xmax.svg @@ -0,0 +1 @@ +ST_XMAX(point) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_xmin.svg b/docs/reference/esql/functions/signature/st_xmin.svg new file mode 100644 index 0000000000000..13d479b0458be --- /dev/null +++ b/docs/reference/esql/functions/signature/st_xmin.svg @@ -0,0 +1 @@ +ST_XMIN(point) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_ymax.svg b/docs/reference/esql/functions/signature/st_ymax.svg new file mode 100644 index 0000000000000..e6ecb00185c84 --- /dev/null +++ b/docs/reference/esql/functions/signature/st_ymax.svg @@ -0,0 +1 @@ +ST_YMAX(point) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_ymin.svg b/docs/reference/esql/functions/signature/st_ymin.svg new file mode 100644 index 0000000000000..ae722f1edc3d4 --- /dev/null +++ b/docs/reference/esql/functions/signature/st_ymin.svg @@ -0,0 +1 @@ +ST_YMIN(point) \ No newline at end of file diff --git a/docs/reference/esql/functions/spatial-functions.asciidoc b/docs/reference/esql/functions/spatial-functions.asciidoc index eee44d337b4c6..c6a8467b39996 100644 --- a/docs/reference/esql/functions/spatial-functions.asciidoc +++ b/docs/reference/esql/functions/spatial-functions.asciidoc @@ -15,6 +15,11 @@ * <> * <> * <> +* experimental:[] <> +* experimental:[] <> +* experimental:[] <> +* experimental:[] <> +* experimental:[] <> // end::spatial_list[] include::layout/st_distance.asciidoc[] @@ -24,3 +29,8 @@ include::layout/st_contains.asciidoc[] include::layout/st_within.asciidoc[] include::layout/st_x.asciidoc[] include::layout/st_y.asciidoc[] +include::layout/st_envelope.asciidoc[] +include::layout/st_xmax.asciidoc[] +include::layout/st_xmin.asciidoc[] +include::layout/st_ymax.asciidoc[] +include::layout/st_ymin.asciidoc[] diff --git a/docs/reference/esql/functions/types/st_envelope.asciidoc b/docs/reference/esql/functions/types/st_envelope.asciidoc new file mode 100644 index 0000000000000..43355394c6015 --- /dev/null +++ b/docs/reference/esql/functions/types/st_envelope.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +geometry | result +cartesian_point | cartesian_shape +cartesian_shape | cartesian_shape +geo_point | geo_shape +geo_shape | geo_shape +|=== diff --git a/docs/reference/esql/functions/types/st_xmax.asciidoc b/docs/reference/esql/functions/types/st_xmax.asciidoc new file mode 100644 index 0000000000000..418c5cafae6f3 --- /dev/null +++ b/docs/reference/esql/functions/types/st_xmax.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +point | result +cartesian_point | double +cartesian_shape | double +geo_point | double +geo_shape | double +|=== diff --git a/docs/reference/esql/functions/types/st_xmin.asciidoc b/docs/reference/esql/functions/types/st_xmin.asciidoc new file mode 100644 index 0000000000000..418c5cafae6f3 --- /dev/null +++ b/docs/reference/esql/functions/types/st_xmin.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +point | result +cartesian_point | double +cartesian_shape | double +geo_point | double +geo_shape | double +|=== diff --git a/docs/reference/esql/functions/types/st_ymax.asciidoc b/docs/reference/esql/functions/types/st_ymax.asciidoc new file mode 100644 index 0000000000000..418c5cafae6f3 --- /dev/null +++ b/docs/reference/esql/functions/types/st_ymax.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +point | result +cartesian_point | double +cartesian_shape | double +geo_point | double +geo_shape | double +|=== diff --git a/docs/reference/esql/functions/types/st_ymin.asciidoc b/docs/reference/esql/functions/types/st_ymin.asciidoc new file mode 100644 index 0000000000000..418c5cafae6f3 --- /dev/null +++ b/docs/reference/esql/functions/types/st_ymin.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +point | result +cartesian_point | double +cartesian_shape | double +geo_point | double +geo_shape | double +|=== diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java new file mode 100644 index 0000000000000..eee4a62c7d588 --- /dev/null +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java @@ -0,0 +1,356 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.geometry.utils; + +import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; + +import java.util.Locale; +import java.util.Optional; + +/** + * This visitor is designed to determine the spatial envelope (or BBOX or MBR) of a potentially complex geometry. + * It has two modes: + *
    + *
  • + * Cartesian mode: The envelope is determined by the minimum and maximum x/y coordinates. + * Incoming BBOX geometries with minX > maxX are treated as invalid. + * Resulting BBOX geometries will always have minX <= maxX. + *
    • + *
  • + * Geographic mode: The envelope is determined by the minimum and maximum x/y coordinates, + * considering the possibility of wrapping the longitude around the dateline. + * A bounding box can be determined either by wrapping the longitude around the dateline or not, + * and the smaller bounding box is chosen. It is possible to disable the wrapping of the longitude. + *
+ * Usage of this is as simple as: + * + * Optional<Rectangle> bbox = SpatialEnvelopeVisitor.visit(geometry); + * if (bbox.isPresent()) { + * Rectangle envelope = bbox.get(); + * // Do stuff with the envelope + * } + * + * It is also possible to create the inner PointVisitor separately, as well as use the visitor for multiple geometries. + * + * PointVisitor pointVisitor = new CartesianPointVisitor(); + * SpatialEnvelopeVisitor visitor = new SpatialEnvelopeVisitor(pointVisitor); + * for (Geometry geometry : geometries) { + * geometry.visit(visitor); + * } + * if (visitor.isValid()) { + * Rectangle envelope = visitor.getResult(); + * // Do stuff with the envelope + * } + * + * Code that wishes to modify the behaviour of the visitor can implement the PointVisitor interface, + * or extend the existing implementations. + */ +public class SpatialEnvelopeVisitor implements GeometryVisitor { + + private final PointVisitor pointVisitor; + + public SpatialEnvelopeVisitor(PointVisitor pointVisitor) { + this.pointVisitor = pointVisitor; + } + + /** + * Determine the BBOX without considering the CRS or wrapping of the longitude. + * Note that incoming BBOX's that do cross the dateline (minx>maxx) will be treated as invalid. + */ + public static Optional visitCartesian(Geometry geometry) { + var visitor = new SpatialEnvelopeVisitor(new CartesianPointVisitor()); + if (geometry.visit(visitor)) { + return Optional.of(visitor.getResult()); + } + return Optional.empty(); + } + + /** + * Determine the BBOX assuming the CRS is geographic (eg WGS84) and optionally wrapping the longitude around the dateline. + */ + public static Optional visitGeo(Geometry geometry, boolean wrapLongitude) { + var visitor = new SpatialEnvelopeVisitor(new GeoPointVisitor(wrapLongitude)); + if (geometry.visit(visitor)) { + return Optional.of(visitor.getResult()); + } + return Optional.empty(); + } + + public Rectangle getResult() { + return pointVisitor.getResult(); + } + + /** + * Visitor for visiting points and rectangles. This is where the actual envelope calculation happens. + * There are two implementations, one for cartesian coordinates and one for geographic coordinates. + * The latter can optionally wrap the longitude around the dateline. + */ + public interface PointVisitor { + void visitPoint(double x, double y); + + void visitRectangle(double minX, double maxX, double maxY, double minY); + + boolean isValid(); + + Rectangle getResult(); + } + + /** + * The cartesian point visitor determines the envelope by the minimum and maximum x/y coordinates. + * It also disallows invalid rectangles where minX > maxX. + */ + public static class CartesianPointVisitor implements PointVisitor { + private double minX = Double.POSITIVE_INFINITY; + private double minY = Double.POSITIVE_INFINITY; + private double maxX = Double.NEGATIVE_INFINITY; + private double maxY = Double.NEGATIVE_INFINITY; + + public double getMinX() { + return minX; + } + + public double getMinY() { + return minY; + } + + public double getMaxX() { + return maxX; + } + + public double getMaxY() { + return maxY; + } + + @Override + public void visitPoint(double x, double y) { + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + + @Override + public void visitRectangle(double minX, double maxX, double maxY, double minY) { + if (minX > maxX) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid cartesian rectangle: minX (%s) > maxX (%s)", minX, maxX) + ); + } + this.minX = Math.min(this.minX, minX); + this.minY = Math.min(this.minY, minY); + this.maxX = Math.max(this.maxX, maxX); + this.maxY = Math.max(this.maxY, maxY); + } + + @Override + public boolean isValid() { + return minY != Double.POSITIVE_INFINITY; + } + + @Override + public Rectangle getResult() { + return new Rectangle(minX, maxX, maxY, minY); + } + } + + /** + * The geographic point visitor determines the envelope by the minimum and maximum x/y coordinates, + * while allowing for wrapping the longitude around the dateline. + * When longitude wrapping is enabled, the visitor will determine the smallest bounding box between the two choices: + *
    + *
  • Wrapping around the front of the earth, in which case the result will have minx < maxx
    • + *
  • Wrapping around the back of the earth, crossing the dateline, in which case the result will have minx > maxx
    • + *
+ */ + public static class GeoPointVisitor implements PointVisitor { + private double minY = Double.POSITIVE_INFINITY; + private double maxY = Double.NEGATIVE_INFINITY; + private double minNegX = Double.POSITIVE_INFINITY; + private double maxNegX = Double.NEGATIVE_INFINITY; + private double minPosX = Double.POSITIVE_INFINITY; + private double maxPosX = Double.NEGATIVE_INFINITY; + + public double getMinY() { + return minY; + } + + public double getMaxY() { + return maxY; + } + + public double getMinNegX() { + return minNegX; + } + + public double getMaxNegX() { + return maxNegX; + } + + public double getMinPosX() { + return minPosX; + } + + public double getMaxPosX() { + return maxPosX; + } + + private final boolean wrapLongitude; + + public GeoPointVisitor(boolean wrapLongitude) { + this.wrapLongitude = wrapLongitude; + } + + @Override + public void visitPoint(double x, double y) { + minY = Math.min(minY, y); + maxY = Math.max(maxY, y); + visitLongitude(x); + } + + @Override + public void visitRectangle(double minX, double maxX, double maxY, double minY) { + this.minY = Math.min(this.minY, minY); + this.maxY = Math.max(this.maxY, maxY); + visitLongitude(minX); + visitLongitude(maxX); + } + + private void visitLongitude(double x) { + if (x >= 0) { + minPosX = Math.min(minPosX, x); + maxPosX = Math.max(maxPosX, x); + } else { + minNegX = Math.min(minNegX, x); + maxNegX = Math.max(maxNegX, x); + } + } + + @Override + public boolean isValid() { + return minY != Double.POSITIVE_INFINITY; + } + + @Override + public Rectangle getResult() { + return getResult(minNegX, minPosX, maxNegX, maxPosX, maxY, minY, wrapLongitude); + } + + private static Rectangle getResult( + double minNegX, + double minPosX, + double maxNegX, + double maxPosX, + double maxY, + double minY, + boolean wrapLongitude + ) { + assert Double.isFinite(maxY); + if (Double.isInfinite(minPosX)) { + return new Rectangle(minNegX, maxNegX, maxY, minY); + } else if (Double.isInfinite(minNegX)) { + return new Rectangle(minPosX, maxPosX, maxY, minY); + } else if (wrapLongitude) { + double unwrappedWidth = maxPosX - minNegX; + double wrappedWidth = (180 - minPosX) - (-180 - maxNegX); + if (unwrappedWidth <= wrappedWidth) { + return new Rectangle(minNegX, maxPosX, maxY, minY); + } else { + return new Rectangle(minPosX, maxNegX, maxY, minY); + } + } else { + return new Rectangle(minNegX, maxPosX, maxY, minY); + } + } + } + + private boolean isValid() { + return pointVisitor.isValid(); + } + + @Override + public Boolean visit(Circle circle) throws RuntimeException { + // TODO: Support circle, if given CRS (needed for radius to x/y coordinate transformation) + throw new UnsupportedOperationException("Circle is not supported"); + } + + @Override + public Boolean visit(GeometryCollection collection) throws RuntimeException { + collection.forEach(geometry -> geometry.visit(this)); + return isValid(); + } + + @Override + public Boolean visit(Line line) throws RuntimeException { + for (int i = 0; i < line.length(); i++) { + pointVisitor.visitPoint(line.getX(i), line.getY(i)); + } + return isValid(); + } + + @Override + public Boolean visit(LinearRing ring) throws RuntimeException { + for (int i = 0; i < ring.length(); i++) { + pointVisitor.visitPoint(ring.getX(i), ring.getY(i)); + } + return isValid(); + } + + @Override + public Boolean visit(MultiLine multiLine) throws RuntimeException { + multiLine.forEach(line -> line.visit(this)); + return isValid(); + } + + @Override + public Boolean visit(MultiPoint multiPoint) throws RuntimeException { + for (int i = 0; i < multiPoint.size(); i++) { + visit(multiPoint.get(i)); + } + return isValid(); + } + + @Override + public Boolean visit(MultiPolygon multiPolygon) throws RuntimeException { + multiPolygon.forEach(polygon -> polygon.visit(this)); + return isValid(); + } + + @Override + public Boolean visit(Point point) throws RuntimeException { + pointVisitor.visitPoint(point.getX(), point.getY()); + return isValid(); + } + + @Override + public Boolean visit(Polygon polygon) throws RuntimeException { + visit(polygon.getPolygon()); + for (int i = 0; i < polygon.getNumberOfHoles(); i++) { + visit(polygon.getHole(i)); + } + return isValid(); + } + + @Override + public Boolean visit(Rectangle rectangle) throws RuntimeException { + pointVisitor.visitRectangle(rectangle.getMinX(), rectangle.getMaxX(), rectangle.getMaxY(), rectangle.getMinY()); + return isValid(); + } +} diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitorTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitorTests.java new file mode 100644 index 0000000000000..fc35df295e566 --- /dev/null +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitorTests.java @@ -0,0 +1,194 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.geometry.utils; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geo.ShapeTestUtils; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class SpatialEnvelopeVisitorTests extends ESTestCase { + + public void testVisitCartesianShape() { + for (int i = 0; i < 1000; i++) { + var geometry = ShapeTestUtils.randomGeometryWithoutCircle(0, false); + var bbox = SpatialEnvelopeVisitor.visitCartesian(geometry); + assertNotNull(bbox); + assertTrue(i + ": " + geometry, bbox.isPresent()); + var result = bbox.get(); + assertThat(i + ": " + geometry, result.getMinX(), lessThanOrEqualTo(result.getMaxX())); + assertThat(i + ": " + geometry, result.getMinY(), lessThanOrEqualTo(result.getMaxY())); + } + } + + public void testVisitGeoShapeNoWrap() { + for (int i = 0; i < 1000; i++) { + var geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, false); + var bbox = SpatialEnvelopeVisitor.visitGeo(geometry, false); + assertNotNull(bbox); + assertTrue(i + ": " + geometry, bbox.isPresent()); + var result = bbox.get(); + assertThat(i + ": " + geometry, result.getMinX(), lessThanOrEqualTo(result.getMaxX())); + assertThat(i + ": " + geometry, result.getMinY(), lessThanOrEqualTo(result.getMaxY())); + } + } + + public void testVisitGeoShapeWrap() { + for (int i = 0; i < 1000; i++) { + var geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, true); + var bbox = SpatialEnvelopeVisitor.visitGeo(geometry, false); + assertNotNull(bbox); + assertTrue(i + ": " + geometry, bbox.isPresent()); + var result = bbox.get(); + assertThat(i + ": " + geometry, result.getMinX(), lessThanOrEqualTo(result.getMaxX())); + assertThat(i + ": " + geometry, result.getMinY(), lessThanOrEqualTo(result.getMaxY())); + } + } + + public void testVisitCartesianPoints() { + var visitor = new SpatialEnvelopeVisitor(new SpatialEnvelopeVisitor.CartesianPointVisitor()); + double minX = Double.MAX_VALUE; + double minY = Double.MAX_VALUE; + double maxX = -Double.MAX_VALUE; + double maxY = -Double.MAX_VALUE; + for (int i = 0; i < 1000; i++) { + var x = randomFloat(); + var y = randomFloat(); + var point = new Point(x, y); + visitor.visit(point); + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + var result = visitor.getResult(); + assertThat(i + ": " + point, result.getMinX(), equalTo(minX)); + assertThat(i + ": " + point, result.getMinY(), equalTo(minY)); + assertThat(i + ": " + point, result.getMaxX(), equalTo(maxX)); + assertThat(i + ": " + point, result.getMaxY(), equalTo(maxY)); + } + } + + public void testVisitGeoPointsNoWrapping() { + var visitor = new SpatialEnvelopeVisitor(new SpatialEnvelopeVisitor.GeoPointVisitor(false)); + double minY = Double.MAX_VALUE; + double maxY = -Double.MAX_VALUE; + double minX = Double.MAX_VALUE; + double maxX = -Double.MAX_VALUE; + for (int i = 0; i < 1000; i++) { + var point = GeometryTestUtils.randomPoint(); + visitor.visit(point); + minY = Math.min(minY, point.getY()); + maxY = Math.max(maxY, point.getY()); + minX = Math.min(minX, point.getX()); + maxX = Math.max(maxX, point.getX()); + var result = visitor.getResult(); + assertThat(i + ": " + point, result.getMinX(), lessThanOrEqualTo(result.getMaxX())); + assertThat(i + ": " + point, result.getMinX(), equalTo(minX)); + assertThat(i + ": " + point, result.getMinY(), equalTo(minY)); + assertThat(i + ": " + point, result.getMaxX(), equalTo(maxX)); + assertThat(i + ": " + point, result.getMaxY(), equalTo(maxY)); + } + } + + public void testVisitGeoPointsWrapping() { + var visitor = new SpatialEnvelopeVisitor(new SpatialEnvelopeVisitor.GeoPointVisitor(true)); + double minY = Double.POSITIVE_INFINITY; + double maxY = Double.NEGATIVE_INFINITY; + double minNegX = Double.POSITIVE_INFINITY; + double maxNegX = Double.NEGATIVE_INFINITY; + double minPosX = Double.POSITIVE_INFINITY; + double maxPosX = Double.NEGATIVE_INFINITY; + for (int i = 0; i < 1000; i++) { + var point = GeometryTestUtils.randomPoint(); + visitor.visit(point); + minY = Math.min(minY, point.getY()); + maxY = Math.max(maxY, point.getY()); + if (point.getX() >= 0) { + minPosX = Math.min(minPosX, point.getX()); + maxPosX = Math.max(maxPosX, point.getX()); + } else { + minNegX = Math.min(minNegX, point.getX()); + maxNegX = Math.max(maxNegX, point.getX()); + } + var result = visitor.getResult(); + if (Double.isInfinite(minPosX)) { + // Only negative x values were considered + assertRectangleResult(i + ": " + point, result, minNegX, maxNegX, maxY, minY, false); + } else if (Double.isInfinite(minNegX)) { + // Only positive x values were considered + assertRectangleResult(i + ": " + point, result, minPosX, maxPosX, maxY, minY, false); + } else { + // Both positive and negative x values exist, we need to decide which way to wrap the bbox + double unwrappedWidth = maxPosX - minNegX; + double wrappedWidth = (180 - minPosX) - (-180 - maxNegX); + if (unwrappedWidth <= wrappedWidth) { + // The smaller bbox is around the front of the planet, no dateline wrapping required + assertRectangleResult(i + ": " + point, result, minNegX, maxPosX, maxY, minY, false); + } else { + // The smaller bbox is around the back of the planet, dateline wrapping required (minx > maxx) + assertRectangleResult(i + ": " + point, result, minPosX, maxNegX, maxY, minY, true); + } + } + } + } + + public void testWillCrossDateline() { + var visitor = new SpatialEnvelopeVisitor(new SpatialEnvelopeVisitor.GeoPointVisitor(true)); + visitor.visit(new Point(-90.0, 0.0)); + visitor.visit(new Point(90.0, 0.0)); + assertCrossesDateline(visitor, false); + visitor.visit(new Point(-89.0, 0.0)); + visitor.visit(new Point(89.0, 0.0)); + assertCrossesDateline(visitor, false); + visitor.visit(new Point(-100.0, 0.0)); + visitor.visit(new Point(100.0, 0.0)); + assertCrossesDateline(visitor, true); + visitor.visit(new Point(-70.0, 0.0)); + visitor.visit(new Point(70.0, 0.0)); + assertCrossesDateline(visitor, false); + visitor.visit(new Point(-120.0, 0.0)); + visitor.visit(new Point(120.0, 0.0)); + assertCrossesDateline(visitor, true); + } + + private void assertCrossesDateline(SpatialEnvelopeVisitor visitor, boolean crossesDateline) { + var result = visitor.getResult(); + if (crossesDateline) { + assertThat("Crosses dateline, minx>maxx", result.getMinX(), greaterThanOrEqualTo(result.getMaxX())); + } else { + assertThat("Does not cross dateline, minx 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendBytesRef(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static BytesRef evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StEnvelope.fromWellKnownBinary(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StEnvelopeFromWKBEvaluator get(DriverContext context) { + return new StEnvelopeFromWKBEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StEnvelopeFromWKBEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeFromWKBGeoEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeFromWKBGeoEvaluator.java new file mode 100644 index 0000000000000..c61e825c0ee71 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeFromWKBGeoEvaluator.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StEnvelope}. + * This class is generated. Do not edit it. + */ +public final class StEnvelopeFromWKBGeoEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StEnvelopeFromWKBGeoEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StEnvelopeFromWKBGeo"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantBytesRefBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendBytesRef(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static BytesRef evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StEnvelope.fromWellKnownBinaryGeo(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + BytesRef value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendBytesRef(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static BytesRef evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StEnvelope.fromWellKnownBinaryGeo(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StEnvelopeFromWKBGeoEvaluator get(DriverContext context) { + return new StEnvelopeFromWKBGeoEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StEnvelopeFromWKBGeoEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBEvaluator.java new file mode 100644 index 0000000000000..0d51ef709c217 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StXMax}. + * This class is generated. Do not edit it. + */ +public final class StXMaxFromWKBEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StXMaxFromWKBEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StXMaxFromWKB"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMax.fromWellKnownBinary(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMax.fromWellKnownBinary(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StXMaxFromWKBEvaluator get(DriverContext context) { + return new StXMaxFromWKBEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StXMaxFromWKBEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBGeoEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBGeoEvaluator.java new file mode 100644 index 0000000000000..3707bf421d550 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBGeoEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StXMax}. + * This class is generated. Do not edit it. + */ +public final class StXMaxFromWKBGeoEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StXMaxFromWKBGeoEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StXMaxFromWKBGeo"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMax.fromWellKnownBinaryGeo(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMax.fromWellKnownBinaryGeo(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StXMaxFromWKBGeoEvaluator get(DriverContext context) { + return new StXMaxFromWKBGeoEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StXMaxFromWKBGeoEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBEvaluator.java new file mode 100644 index 0000000000000..699402ad68dee --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StXMin}. + * This class is generated. Do not edit it. + */ +public final class StXMinFromWKBEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StXMinFromWKBEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StXMinFromWKB"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMin.fromWellKnownBinary(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMin.fromWellKnownBinary(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StXMinFromWKBEvaluator get(DriverContext context) { + return new StXMinFromWKBEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StXMinFromWKBEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBGeoEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBGeoEvaluator.java new file mode 100644 index 0000000000000..6a8c041595c1c --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBGeoEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StXMin}. + * This class is generated. Do not edit it. + */ +public final class StXMinFromWKBGeoEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StXMinFromWKBGeoEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StXMinFromWKBGeo"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMin.fromWellKnownBinaryGeo(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMin.fromWellKnownBinaryGeo(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StXMinFromWKBGeoEvaluator get(DriverContext context) { + return new StXMinFromWKBGeoEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StXMinFromWKBGeoEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBEvaluator.java new file mode 100644 index 0000000000000..e8b50099f38f6 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StYMax}. + * This class is generated. Do not edit it. + */ +public final class StYMaxFromWKBEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StYMaxFromWKBEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StYMaxFromWKB"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMax.fromWellKnownBinary(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMax.fromWellKnownBinary(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StYMaxFromWKBEvaluator get(DriverContext context) { + return new StYMaxFromWKBEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StYMaxFromWKBEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBGeoEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBGeoEvaluator.java new file mode 100644 index 0000000000000..00e75f862a86c --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBGeoEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StYMax}. + * This class is generated. Do not edit it. + */ +public final class StYMaxFromWKBGeoEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StYMaxFromWKBGeoEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StYMaxFromWKBGeo"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMax.fromWellKnownBinaryGeo(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMax.fromWellKnownBinaryGeo(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StYMaxFromWKBGeoEvaluator get(DriverContext context) { + return new StYMaxFromWKBGeoEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StYMaxFromWKBGeoEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBEvaluator.java new file mode 100644 index 0000000000000..cab66683261aa --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StYMin}. + * This class is generated. Do not edit it. + */ +public final class StYMinFromWKBEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StYMinFromWKBEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StYMinFromWKB"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMin.fromWellKnownBinary(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMin.fromWellKnownBinary(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StYMinFromWKBEvaluator get(DriverContext context) { + return new StYMinFromWKBEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StYMinFromWKBEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBGeoEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBGeoEvaluator.java new file mode 100644 index 0000000000000..8bae9d369fbb4 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBGeoEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StYMin}. + * This class is generated. Do not edit it. + */ +public final class StYMinFromWKBGeoEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StYMinFromWKBGeoEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StYMinFromWKBGeo"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMin.fromWellKnownBinaryGeo(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMin.fromWellKnownBinaryGeo(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StYMinFromWKBGeoEvaluator get(DriverContext context) { + return new StYMinFromWKBGeoEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StYMinFromWKBGeoEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 4845c7061949b..19ba6a5151eaf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -214,6 +214,11 @@ public enum Cap { */ SPATIAL_CENTROID_NO_RECORDS, + /** + * Support ST_ENVELOPE function (and related ST_XMIN, etc.). + */ + ST_ENVELOPE, + /** * Fix to GROK and DISSECT that allows extracting attributes with the same name as the input * https://github.com/elastic/elasticsearch/issues/110184 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java index 7e2de0094c2ab..febeccdad9d78 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java @@ -57,8 +57,13 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialIntersects; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialWithin; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StEnvelope; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StX; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StXMax; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StXMin; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StY; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StYMax; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StYMin; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ByteLength; import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; @@ -166,6 +171,11 @@ public static List unaryScalars() { entries.add(Sinh.ENTRY); entries.add(Space.ENTRY); entries.add(Sqrt.ENTRY); + entries.add(StEnvelope.ENTRY); + entries.add(StXMax.ENTRY); + entries.add(StXMin.ENTRY); + entries.add(StYMax.ENTRY); + entries.add(StYMin.ENTRY); entries.add(StX.ENTRY); entries.add(StY.ENTRY); entries.add(Tan.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 37b159922906c..c66a5293eb14a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -117,8 +117,13 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialIntersects; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialWithin; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StEnvelope; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StX; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StXMax; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StXMin; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StY; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StYMax; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StYMin; import org.elasticsearch.xpack.esql.expression.function.scalar.string.BitLength; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ByteLength; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; @@ -351,6 +356,11 @@ private static FunctionDefinition[][] functions() { def(SpatialIntersects.class, SpatialIntersects::new, "st_intersects"), def(SpatialWithin.class, SpatialWithin::new, "st_within"), def(StDistance.class, StDistance::new, "st_distance"), + def(StEnvelope.class, StEnvelope::new, "st_envelope"), + def(StXMax.class, StXMax::new, "st_xmax"), + def(StXMin.class, StXMin::new, "st_xmin"), + def(StYMax.class, StYMax::new, "st_ymax"), + def(StYMin.class, StYMin::new, "st_ymin"), def(StX.class, StX::new, "st_x"), def(StY.class, StY::new, "st_y") }, // conditional diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelope.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelope.java new file mode 100644 index 0000000000000..934991f3a8088 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelope.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; +import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial; + +/** + * Determines the minimum bounding rectangle of a geometry. + * The function `st_envelope` is defined in the OGC Simple Feature Access standard. + * Alternatively it is well described in PostGIS documentation at + * PostGIS:ST_ENVELOPE. + */ +public class StEnvelope extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "StEnvelope", + StEnvelope::new + ); + private DataType dataType; + + @FunctionInfo( + returnType = { "geo_shape", "cartesian_shape" }, + description = "Determines the minimum bounding box of the supplied geometry.", + examples = @Example(file = "spatial_shapes", tag = "st_envelope") + ) + public StEnvelope( + Source source, + @Param( + name = "geometry", + type = { "geo_point", "geo_shape", "cartesian_point", "cartesian_shape" }, + description = "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. " + + "If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private StEnvelope(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + var resolution = isSpatial(field(), sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + if (resolution.resolved()) { + this.dataType = switch (field().dataType()) { + case GEO_POINT, GEO_SHAPE -> GEO_SHAPE; + case CARTESIAN_POINT, CARTESIAN_SHAPE -> CARTESIAN_SHAPE; + default -> NULL; + }; + } + return resolution; + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + if (field().dataType() == GEO_POINT || field().dataType() == DataType.GEO_SHAPE) { + return new StEnvelopeFromWKBGeoEvaluator.Factory(toEvaluator.apply(field()), source()); + } + return new StEnvelopeFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); + } + + @Override + public DataType dataType() { + return dataType; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new StEnvelope(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StEnvelope::new, field()); + } + + @ConvertEvaluator(extraName = "FromWKB", warnExceptions = { IllegalArgumentException.class }) + static BytesRef fromWellKnownBinary(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point) { + return wkb; + } + var envelope = SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return UNSPECIFIED.asWkb(envelope.get()); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } + + @ConvertEvaluator(extraName = "FromWKBGeo", warnExceptions = { IllegalArgumentException.class }) + static BytesRef fromWellKnownBinaryGeo(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point) { + return wkb; + } + var envelope = SpatialEnvelopeVisitor.visitGeo(geometry, true); + if (envelope.isPresent()) { + return UNSPECIFIED.asWkb(envelope.get()); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMax.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMax.java new file mode 100644 index 0000000000000..d6d710b175113 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMax.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; +import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial; + +/** + * Determines the maximum value of the x-coordinate from a geometry. + * The function `st_xmax` is defined in the OGC Simple Feature Access standard. + * Alternatively it is well described in PostGIS documentation at PostGIS:ST_XMAX. + */ +public class StXMax extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "StXMax", StXMax::new); + + @FunctionInfo( + returnType = "double", + description = "Extracts the maximum value of the `x` coordinates from the supplied geometry.\n" + + "If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `longitude` value.", + examples = @Example(file = "spatial_shapes", tag = "st_x_y_min_max") + ) + public StXMax( + Source source, + @Param( + name = "point", + type = { "geo_point", "geo_shape", "cartesian_point", "cartesian_shape" }, + description = "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. " + + "If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private StXMax(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + return isSpatial(field(), sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + if (field().dataType() == GEO_POINT || field().dataType() == DataType.GEO_SHAPE) { + return new StXMaxFromWKBGeoEvaluator.Factory(toEvaluator.apply(field()), source()); + } + return new StXMaxFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); + } + + @Override + public DataType dataType() { + return DOUBLE; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new StXMax(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StXMax::new, field()); + } + + @ConvertEvaluator(extraName = "FromWKB", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinary(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getX(); + } + var envelope = SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMaxX(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } + + @ConvertEvaluator(extraName = "FromWKBGeo", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinaryGeo(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getX(); + } + var envelope = SpatialEnvelopeVisitor.visitGeo(geometry, true); + if (envelope.isPresent()) { + return envelope.get().getMaxX(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMin.java new file mode 100644 index 0000000000000..a5fa11bc11b0f --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMin.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; +import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial; + +/** + * Determines the minimum value of the x-coordinate from a geometry. + * The function `st_xmin` is defined in the OGC Simple Feature Access standard. + * Alternatively it is well described in PostGIS documentation at PostGIS:ST_XMIN. + */ +public class StXMin extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "StXMin", StXMin::new); + + @FunctionInfo( + returnType = "double", + description = "Extracts the minimum value of the `x` coordinates from the supplied geometry.\n" + + "If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `longitude` value.", + examples = @Example(file = "spatial_shapes", tag = "st_x_y_min_max") + ) + public StXMin( + Source source, + @Param( + name = "point", + type = { "geo_point", "geo_shape", "cartesian_point", "cartesian_shape" }, + description = "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. " + + "If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private StXMin(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + return isSpatial(field(), sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + if (field().dataType() == GEO_POINT || field().dataType() == DataType.GEO_SHAPE) { + return new StXMinFromWKBGeoEvaluator.Factory(toEvaluator.apply(field()), source()); + } + return new StXMinFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); + } + + @Override + public DataType dataType() { + return DOUBLE; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new StXMin(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StXMin::new, field()); + } + + @ConvertEvaluator(extraName = "FromWKB", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinary(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getX(); + } + var envelope = SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMinX(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } + + @ConvertEvaluator(extraName = "FromWKBGeo", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinaryGeo(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getX(); + } + var envelope = SpatialEnvelopeVisitor.visitGeo(geometry, true); + if (envelope.isPresent()) { + return envelope.get().getMinX(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMax.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMax.java new file mode 100644 index 0000000000000..fbbea8e024a6b --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMax.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; +import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial; + +/** + * Determines the maximum value of the y-coordinate from a geometry. + * The function `st_ymax` is defined in the OGC Simple Feature Access standard. + * Alternatively it is well described in PostGIS documentation at PostGIS:ST_YMAX. + */ +public class StYMax extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "StYMax", StYMax::new); + + @FunctionInfo( + returnType = "double", + description = "Extracts the maximum value of the `y` coordinates from the supplied geometry.\n" + + "If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `latitude` value.", + examples = @Example(file = "spatial_shapes", tag = "st_x_y_min_max") + ) + public StYMax( + Source source, + @Param( + name = "point", + type = { "geo_point", "geo_shape", "cartesian_point", "cartesian_shape" }, + description = "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. " + + "If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private StYMax(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + return isSpatial(field(), sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + if (field().dataType() == GEO_POINT || field().dataType() == DataType.GEO_SHAPE) { + return new StYMaxFromWKBGeoEvaluator.Factory(toEvaluator.apply(field()), source()); + } + return new StYMaxFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); + } + + @Override + public DataType dataType() { + return DOUBLE; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new StYMax(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StYMax::new, field()); + } + + @ConvertEvaluator(extraName = "FromWKB", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinary(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getY(); + } + var envelope = SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMaxY(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } + + @ConvertEvaluator(extraName = "FromWKBGeo", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinaryGeo(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getY(); + } + var envelope = SpatialEnvelopeVisitor.visitGeo(geometry, true); + if (envelope.isPresent()) { + return envelope.get().getMaxY(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMin.java new file mode 100644 index 0000000000000..1707d3b4f2fb9 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMin.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; +import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial; + +/** + * Determines the minimum value of the y-coordinate from a geometry. + * The function `st_ymin` is defined in the OGC Simple Feature Access standard. + * Alternatively it is well described in PostGIS documentation at PostGIS:ST_YMIN. + */ +public class StYMin extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "StYMin", StYMin::new); + + @FunctionInfo( + returnType = "double", + description = "Extracts the minimum value of the `y` coordinates from the supplied geometry.\n" + + "If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `latitude` value.", + examples = @Example(file = "spatial_shapes", tag = "st_x_y_min_max") + ) + public StYMin( + Source source, + @Param( + name = "point", + type = { "geo_point", "geo_shape", "cartesian_point", "cartesian_shape" }, + description = "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. " + + "If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private StYMin(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + return isSpatial(field(), sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + if (field().dataType() == GEO_POINT || field().dataType() == DataType.GEO_SHAPE) { + return new StYMinFromWKBGeoEvaluator.Factory(toEvaluator.apply(field()), source()); + } + return new StYMinFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); + } + + @Override + public DataType dataType() { + return DOUBLE; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new StYMin(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StYMin::new, field()); + } + + @ConvertEvaluator(extraName = "FromWKB", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinary(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getY(); + } + var envelope = SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMinY(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } + + @ConvertEvaluator(extraName = "FromWKBGeo", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinaryGeo(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getY(); + } + var envelope = SpatialEnvelopeVisitor.visitGeo(geometry, true); + if (envelope.isPresent()) { + return envelope.get().getMinY(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeTests.java new file mode 100644 index 0000000000000..ac87d45491447 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeTests.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; + +@FunctionName("st_envelope") +public class StEnvelopeTests extends AbstractScalarFunctionTestCase { + public StEnvelopeTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + String expectedGeo = "StEnvelopeFromWKBGeoEvaluator[field=Attribute[channel=0]]"; + String expectedCartesian = "StEnvelopeFromWKBEvaluator[field=Attribute[channel=0]]"; + final List suppliers = new ArrayList<>(); + TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedGeo, GEO_SHAPE, StEnvelopeTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianPoint( + suppliers, + expectedCartesian, + CARTESIAN_SHAPE, + StEnvelopeTests::valueOfCartesian, + List.of() + ); + TestCaseSupplier.forUnaryGeoShape(suppliers, expectedGeo, GEO_SHAPE, StEnvelopeTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianShape( + suppliers, + expectedCartesian, + CARTESIAN_SHAPE, + StEnvelopeTests::valueOfCartesian, + List.of() + ); + return parameterSuppliersFromTypedDataWithDefaultChecks( + false, + suppliers, + (v, p) -> "geo_point, cartesian_point, geo_shape or cartesian_shape" + ); + } + + private static BytesRef valueOfGeo(BytesRef wkb) { + return valueOf(wkb, true); + } + + private static BytesRef valueOfCartesian(BytesRef wkb) { + return valueOf(wkb, false); + } + + private static BytesRef valueOf(BytesRef wkb, boolean geo) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point) { + return wkb; + } + var envelope = geo ? SpatialEnvelopeVisitor.visitGeo(geometry, true) : SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return UNSPECIFIED.asWkb(envelope.get()); + } + throw new IllegalArgumentException("Geometry is empty"); + } + + @Override + protected Expression build(Source source, List args) { + return new StEnvelope(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxTests.java new file mode 100644 index 0000000000000..dc6e61e44f599 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; + +@FunctionName("st_xmax") +public class StXMaxTests extends AbstractScalarFunctionTestCase { + public StXMaxTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + String expectedGeo = "StXMaxFromWKBGeoEvaluator[field=Attribute[channel=0]]"; + String expectedCartesian = "StXMaxFromWKBEvaluator[field=Attribute[channel=0]]"; + final List suppliers = new ArrayList<>(); + TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedGeo, DOUBLE, StXMaxTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianPoint(suppliers, expectedCartesian, DOUBLE, StXMaxTests::valueOfCartesian, List.of()); + TestCaseSupplier.forUnaryGeoShape(suppliers, expectedGeo, DOUBLE, StXMaxTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianShape(suppliers, expectedCartesian, DOUBLE, StXMaxTests::valueOfCartesian, List.of()); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "geo_point, cartesian_point, geo_shape or cartesian_shape" + ); + } + + private static double valueOfGeo(BytesRef wkb) { + return valueOf(wkb, true); + } + + private static double valueOfCartesian(BytesRef wkb) { + return valueOf(wkb, false); + } + + private static double valueOf(BytesRef wkb, boolean geo) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getX(); + } + var envelope = geo ? SpatialEnvelopeVisitor.visitGeo(geometry, true) : SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMaxX(); + } + throw new IllegalArgumentException("Geometry is empty"); + } + + @Override + protected Expression build(Source source, List args) { + return new StXMax(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinTests.java new file mode 100644 index 0000000000000..8c06d18b1e281 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; + +@FunctionName("st_xmin") +public class StXMinTests extends AbstractScalarFunctionTestCase { + public StXMinTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + String expectedGeo = "StXMinFromWKBGeoEvaluator[field=Attribute[channel=0]]"; + String expectedCartesian = "StXMinFromWKBEvaluator[field=Attribute[channel=0]]"; + final List suppliers = new ArrayList<>(); + TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedGeo, DOUBLE, StXMinTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianPoint(suppliers, expectedCartesian, DOUBLE, StXMinTests::valueOfCartesian, List.of()); + TestCaseSupplier.forUnaryGeoShape(suppliers, expectedGeo, DOUBLE, StXMinTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianShape(suppliers, expectedCartesian, DOUBLE, StXMinTests::valueOfCartesian, List.of()); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "geo_point, cartesian_point, geo_shape or cartesian_shape" + ); + } + + private static double valueOfGeo(BytesRef wkb) { + return valueOf(wkb, true); + } + + private static double valueOfCartesian(BytesRef wkb) { + return valueOf(wkb, false); + } + + private static double valueOf(BytesRef wkb, boolean geo) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getX(); + } + var envelope = geo ? SpatialEnvelopeVisitor.visitGeo(geometry, true) : SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMinX(); + } + throw new IllegalArgumentException("Geometry is empty"); + } + + @Override + protected Expression build(Source source, List args) { + return new StXMin(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxTests.java new file mode 100644 index 0000000000000..7222d7517f7ff --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; + +@FunctionName("st_ymax") +public class StYMaxTests extends AbstractScalarFunctionTestCase { + public StYMaxTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + String expectedGeo = "StYMaxFromWKBGeoEvaluator[field=Attribute[channel=0]]"; + String expectedCartesian = "StYMaxFromWKBEvaluator[field=Attribute[channel=0]]"; + final List suppliers = new ArrayList<>(); + TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedGeo, DOUBLE, StYMaxTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianPoint(suppliers, expectedCartesian, DOUBLE, StYMaxTests::valueOfCartesian, List.of()); + TestCaseSupplier.forUnaryGeoShape(suppliers, expectedGeo, DOUBLE, StYMaxTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianShape(suppliers, expectedCartesian, DOUBLE, StYMaxTests::valueOfCartesian, List.of()); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "geo_point, cartesian_point, geo_shape or cartesian_shape" + ); + } + + private static double valueOfGeo(BytesRef wkb) { + return valueOf(wkb, true); + } + + private static double valueOfCartesian(BytesRef wkb) { + return valueOf(wkb, false); + } + + private static double valueOf(BytesRef wkb, boolean geo) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getY(); + } + var envelope = geo ? SpatialEnvelopeVisitor.visitGeo(geometry, true) : SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMaxY(); + } + throw new IllegalArgumentException("Geometry is empty"); + } + + @Override + protected Expression build(Source source, List args) { + return new StYMax(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinTests.java new file mode 100644 index 0000000000000..843c7bb649114 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; + +@FunctionName("st_ymin") +public class StYMinTests extends AbstractScalarFunctionTestCase { + public StYMinTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + String expectedGeo = "StYMinFromWKBGeoEvaluator[field=Attribute[channel=0]]"; + String expectedCartesian = "StYMinFromWKBEvaluator[field=Attribute[channel=0]]"; + final List suppliers = new ArrayList<>(); + TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedGeo, DOUBLE, StYMinTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianPoint(suppliers, expectedCartesian, DOUBLE, StYMinTests::valueOfCartesian, List.of()); + TestCaseSupplier.forUnaryGeoShape(suppliers, expectedGeo, DOUBLE, StYMinTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianShape(suppliers, expectedCartesian, DOUBLE, StYMinTests::valueOfCartesian, List.of()); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "geo_point, cartesian_point, geo_shape or cartesian_shape" + ); + } + + private static double valueOfGeo(BytesRef wkb) { + return valueOf(wkb, true); + } + + private static double valueOfCartesian(BytesRef wkb) { + return valueOf(wkb, false); + } + + private static double valueOf(BytesRef wkb, boolean geo) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getY(); + } + var envelope = geo ? SpatialEnvelopeVisitor.visitGeo(geometry, true) : SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMinY(); + } + throw new IllegalArgumentException("Geometry is empty"); + } + + @Override + protected Expression build(Source source, List args) { + return new StYMin(source, args.get(0)); + } +} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index c23b44c00bd14..26e3c8ed0ef47 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -92,7 +92,7 @@ setup: - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} # Testing for the entire function set isn't feasbile, so we just check that we return the correct count as an approximation. - - length: {esql.functions: 122} # check the "sister" test below for a likely update to the same esql.functions length check + - length: {esql.functions: 127} # check the "sister" test below for a likely update to the same esql.functions length check --- "Basic ESQL usage output (telemetry) non-snapshot version": @@ -163,4 +163,4 @@ setup: - match: {esql.functions.cos: $functions_cos} - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} - - length: {esql.functions: 119} # check the "sister" test above for a likely update to the same esql.functions length check + - length: {esql.functions: 123} # check the "sister" test above for a likely update to the same esql.functions length check From cd58030d46640e22c6e1a736256b5c2ad094cf89 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:05:28 +1100 Subject: [PATCH 006/204] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT test {lookup-join.LookupIPFromIndexKeep ASYNC} #117974 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 17a7b26d1c091..1568bed8a4f30 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -244,6 +244,9 @@ tests: - class: org.elasticsearch.xpack.ml.integration.RegressionIT method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet issue: https://github.com/elastic/elasticsearch/issues/117805 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {lookup-join.LookupIPFromIndexKeep ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/117974 # Examples: # From 4389059bec3c3a2a0a267300b74fc7f7b7716394 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:05:43 +1100 Subject: [PATCH 007/204] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT test {lookup-join.LookupMessageFromIndex ASYNC} #117975 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 1568bed8a4f30..e0d457b26cc73 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -247,6 +247,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT method: test {lookup-join.LookupIPFromIndexKeep ASYNC} issue: https://github.com/elastic/elasticsearch/issues/117974 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {lookup-join.LookupMessageFromIndex ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/117975 # Examples: # From 91d0539c2a1f5926f083c7aee050b5e1b18b8566 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:05:56 +1100 Subject: [PATCH 008/204] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT test {lookup-join.LookupMessageFromIndexKeep ASYNC} #117976 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e0d457b26cc73..944ee0a4ae158 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -250,6 +250,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT method: test {lookup-join.LookupMessageFromIndex ASYNC} issue: https://github.com/elastic/elasticsearch/issues/117975 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {lookup-join.LookupMessageFromIndexKeep ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/117976 # Examples: # From 8134d0244ccac43a826d5282c380f68c5863dab3 Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Wed, 4 Dec 2024 13:13:21 +0100 Subject: [PATCH 009/204] Unmute #117893 (#117909) This test was muted erroneously. --- muted-tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 944ee0a4ae158..c3fe9425def2d 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -221,8 +221,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/117815 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT issue: https://github.com/elastic/elasticsearch/issues/111319 -- class: org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT - issue: https://github.com/elastic/elasticsearch/issues/117893 - class: org.elasticsearch.xpack.core.ml.search.SparseVectorQueryBuilderTests method: testToQuery issue: https://github.com/elastic/elasticsearch/issues/117904 From 435a0cc2721a23f6c97e5102e86b5babf3b64b47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Wed, 4 Dec 2024 13:23:27 +0100 Subject: [PATCH 010/204] CreateClassLoaderEntitlement + extensions to parse logic (#117754) --- .../policy/CreateClassLoaderEntitlement.java | 16 ++++ .../runtime/policy/PolicyParser.java | 81 ++++++++++++------- .../policy/PolicyParserFailureTests.java | 7 +- .../runtime/policy/PolicyParserTests.java | 39 +++++++++ 4 files changed, 109 insertions(+), 34 deletions(-) create mode 100644 libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java new file mode 100644 index 0000000000000..708e0b87711fe --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +public class CreateClassLoaderEntitlement implements Entitlement { + @ExternalEntitlement + public CreateClassLoaderEntitlement() {} + +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java index ea6603af99925..0d1a7c14ece4b 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java @@ -19,22 +19,43 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; - -import static org.elasticsearch.entitlement.runtime.policy.PolicyParserException.newPolicyParserException; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * A parser to parse policy files for entitlements. */ public class PolicyParser { - protected static final String entitlementPackageName = Entitlement.class.getPackage().getName(); + private static final Map> EXTERNAL_ENTITLEMENTS = Stream.of(FileEntitlement.class, CreateClassLoaderEntitlement.class) + .collect(Collectors.toUnmodifiableMap(PolicyParser::getEntitlementTypeName, Function.identity())); protected final XContentParser policyParser; protected final String policyName; + static String getEntitlementTypeName(Class entitlementClass) { + var entitlementClassName = entitlementClass.getSimpleName(); + + if (entitlementClassName.endsWith("Entitlement") == false) { + throw new IllegalArgumentException( + entitlementClassName + " is not a valid Entitlement class name. A valid class name must end with 'Entitlement'" + ); + } + + var strippedClassName = entitlementClassName.substring(0, entitlementClassName.indexOf("Entitlement")); + return Arrays.stream(strippedClassName.split("(?=\\p{Lu})")) + .filter(Predicate.not(String::isEmpty)) + .map(s -> s.toLowerCase(Locale.ROOT)) + .collect(Collectors.joining("_")); + } + public PolicyParser(InputStream inputStream, String policyName) throws IOException { this.policyParser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, Objects.requireNonNull(inputStream)); this.policyName = policyName; @@ -67,18 +88,23 @@ protected Scope parseScope(String scopeName) throws IOException { } List entitlements = new ArrayList<>(); while (policyParser.nextToken() != XContentParser.Token.END_ARRAY) { - if (policyParser.currentToken() != XContentParser.Token.START_OBJECT) { - throw newPolicyParserException(scopeName, "expected object "); - } - if (policyParser.nextToken() != XContentParser.Token.FIELD_NAME) { + if (policyParser.currentToken() == XContentParser.Token.VALUE_STRING) { + String entitlementType = policyParser.text(); + Entitlement entitlement = parseEntitlement(scopeName, entitlementType); + entitlements.add(entitlement); + } else if (policyParser.currentToken() == XContentParser.Token.START_OBJECT) { + if (policyParser.nextToken() != XContentParser.Token.FIELD_NAME) { + throw newPolicyParserException(scopeName, "expected object "); + } + String entitlementType = policyParser.currentName(); + Entitlement entitlement = parseEntitlement(scopeName, entitlementType); + entitlements.add(entitlement); + if (policyParser.nextToken() != XContentParser.Token.END_OBJECT) { + throw newPolicyParserException(scopeName, "expected closing object"); + } + } else { throw newPolicyParserException(scopeName, "expected object "); } - String entitlementType = policyParser.currentName(); - Entitlement entitlement = parseEntitlement(scopeName, entitlementType); - entitlements.add(entitlement); - if (policyParser.nextToken() != XContentParser.Token.END_OBJECT) { - throw newPolicyParserException(scopeName, "expected closing object"); - } } return new Scope(scopeName, entitlements); } catch (IOException ioe) { @@ -87,34 +113,29 @@ protected Scope parseScope(String scopeName) throws IOException { } protected Entitlement parseEntitlement(String scopeName, String entitlementType) throws IOException { - Class entitlementClass; - try { - entitlementClass = Class.forName( - entitlementPackageName - + "." - + Character.toUpperCase(entitlementType.charAt(0)) - + entitlementType.substring(1) - + "Entitlement" - ); - } catch (ClassNotFoundException cnfe) { - throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); - } - if (Entitlement.class.isAssignableFrom(entitlementClass) == false) { + Class entitlementClass = EXTERNAL_ENTITLEMENTS.get(entitlementType); + + if (entitlementClass == null) { throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); } + Constructor entitlementConstructor = entitlementClass.getConstructors()[0]; ExternalEntitlement entitlementMetadata = entitlementConstructor.getAnnotation(ExternalEntitlement.class); if (entitlementMetadata == null) { throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); } - if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) { - throw newPolicyParserException(scopeName, entitlementType, "expected entitlement parameters"); + Class[] parameterTypes = entitlementConstructor.getParameterTypes(); + String[] parametersNames = entitlementMetadata.parameterNames(); + + if (parameterTypes.length != 0 || parametersNames.length != 0) { + if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) { + throw newPolicyParserException(scopeName, entitlementType, "expected entitlement parameters"); + } } + Map parsedValues = policyParser.map(); - Class[] parameterTypes = entitlementConstructor.getParameterTypes(); - String[] parametersNames = entitlementMetadata.parameterNames(); Object[] parameterValues = new Object[parameterTypes.length]; for (int parameterIndex = 0; parameterIndex < parameterTypes.length; ++parameterIndex) { String parameterName = parametersNames[parameterIndex]; diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java index de8280ea87fe5..7eb2b1fb476b3 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.test.ESTestCase; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.nio.charset.StandardCharsets; public class PolicyParserFailureTests extends ESTestCase { @@ -26,7 +25,7 @@ public void testParserSyntaxFailures() { assertEquals("[1:1] policy parsing error for [test-failure-policy.yaml]: expected object ", ppe.getMessage()); } - public void testEntitlementDoesNotExist() throws IOException { + public void testEntitlementDoesNotExist() { PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: - does_not_exist: {} @@ -38,7 +37,7 @@ public void testEntitlementDoesNotExist() throws IOException { ); } - public void testEntitlementMissingParameter() throws IOException { + public void testEntitlementMissingParameter() { PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: - file: {} @@ -61,7 +60,7 @@ public void testEntitlementMissingParameter() throws IOException { ); } - public void testEntitlementExtraneousParameter() throws IOException { + public void testEntitlementExtraneousParameter() { PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: - file: diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java index 40016b2e3027e..a514cfe418895 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java @@ -11,11 +11,31 @@ import org.elasticsearch.test.ESTestCase; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.List; +import static org.elasticsearch.test.LambdaMatchers.transformedMatch; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + public class PolicyParserTests extends ESTestCase { + private static class TestWrongEntitlementName implements Entitlement {} + + public void testGetEntitlementTypeName() { + assertEquals("create_class_loader", PolicyParser.getEntitlementTypeName(CreateClassLoaderEntitlement.class)); + + var ex = expectThrows(IllegalArgumentException.class, () -> PolicyParser.getEntitlementTypeName(TestWrongEntitlementName.class)); + assertThat( + ex.getMessage(), + equalTo("TestWrongEntitlementName is not a valid Entitlement class name. A valid class name must end with 'Entitlement'") + ); + } + public void testPolicyBuilder() throws IOException { Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml") .parsePolicy(); @@ -25,4 +45,23 @@ public void testPolicyBuilder() throws IOException { ); assertEquals(parsedPolicy, builtPolicy); } + + public void testParseCreateClassloader() throws IOException { + Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - create_class_loader + """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml").parsePolicy(); + Policy builtPolicy = new Policy( + "test-policy.yaml", + List.of(new Scope("entitlement-module-name", List.of(new CreateClassLoaderEntitlement()))) + ); + assertThat( + parsedPolicy.scopes, + contains( + both(transformedMatch((Scope scope) -> scope.name, equalTo("entitlement-module-name"))).and( + transformedMatch(scope -> scope.entitlements, contains(instanceOf(CreateClassLoaderEntitlement.class))) + ) + ) + ); + } } From c09bdfb5a0215257e8c8cd1f7ef8ffa345503f56 Mon Sep 17 00:00:00 2001 From: Luke Whiting Date: Wed, 4 Dec 2024 12:27:42 +0000 Subject: [PATCH 011/204] Watcher history index has too many indexed fields - (#71479) (#117701) * Exclude result.input.chain from watcher history index mappings * Update docs/changelog/117701.yaml * Fixup text now fields are disabled higher up the chain * Revert priority change --- docs/changelog/117701.yaml | 6 ++++++ .../support/WatcherIndexTemplateRegistryField.java | 3 ++- .../src/main/resources/watch-history-no-ilm.json | 9 +++++++++ .../src/main/resources/watch-history.json | 9 +++++++++ .../test/integration/HistoryIntegrationTests.java | 6 +++--- 5 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/117701.yaml diff --git a/docs/changelog/117701.yaml b/docs/changelog/117701.yaml new file mode 100644 index 0000000000000..5a72bdeb143e6 --- /dev/null +++ b/docs/changelog/117701.yaml @@ -0,0 +1,6 @@ +pr: 117701 +summary: Watcher history index has too many indexed fields - +area: Watcher +type: bug +issues: + - 71479 diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java index 20dcb84dffe3f..098549029e0ce 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java @@ -22,8 +22,9 @@ public final class WatcherIndexTemplateRegistryField { // version 14: move watch history to data stream // version 15: remove watches and triggered watches, these are now system indices // version 16: change watch history ILM policy + // version 17: exclude input chain from indexing // Note: if you change this, also inform the kibana team around the watcher-ui - public static final int INDEX_TEMPLATE_VERSION = 16; + public static final int INDEX_TEMPLATE_VERSION = 17; public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION; public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION; public static final String[] TEMPLATE_NAMES = new String[] { HISTORY_TEMPLATE_NAME }; diff --git a/x-pack/plugin/core/template-resources/src/main/resources/watch-history-no-ilm.json b/x-pack/plugin/core/template-resources/src/main/resources/watch-history-no-ilm.json index 2eed69c7c58e6..da459cda13463 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/watch-history-no-ilm.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/watch-history-no-ilm.json @@ -54,6 +54,15 @@ "enabled": false } } + }, + { + "disabled_result_input_chain_fields": { + "path_match": "result.input.chain", + "mapping": { + "type": "object", + "enabled": false + } + } } ], "dynamic": false, diff --git a/x-pack/plugin/core/template-resources/src/main/resources/watch-history.json b/x-pack/plugin/core/template-resources/src/main/resources/watch-history.json index 19e4dc022daa1..2abf6570d1f8e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/watch-history.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/watch-history.json @@ -55,6 +55,15 @@ "enabled": false } } + }, + { + "disabled_result_input_chain_fields": { + "path_match": "result.input.chain", + "mapping": { + "type": "object", + "enabled": false + } + } } ], "dynamic": false, diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java index 0070554d99d27..1bcdd060994ce 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java @@ -130,7 +130,7 @@ public void testFailedInputResultWithDotsInFieldNameGetsStored() throws Exceptio String chainedPath = SINGLE_MAPPING_NAME + ".properties.result.properties.input.properties.chain.properties.chained.properties.search" + ".properties.request.properties.body.enabled"; - assertThat(source.getValue(chainedPath), is(false)); + assertThat(source.getValue(chainedPath), nullValue()); } else { String path = SINGLE_MAPPING_NAME + ".properties.result.properties.input.properties.search.properties.request.properties.body.enabled"; @@ -168,11 +168,11 @@ public void testPayloadInputWithDotsInFieldNameWorks() throws Exception { XContentType.JSON ); - // lets make sure the body fields are disabled + // let's make sure the body fields are disabled or, in the case of chained, the whole object is not indexed if (useChained) { String path = SINGLE_MAPPING_NAME + ".properties.result.properties.input.properties.chain.properties.chained.properties.payload.enabled"; - assertThat(source.getValue(path), is(false)); + assertThat(source.getValue(path), nullValue()); } else { String path = SINGLE_MAPPING_NAME + ".properties.result.properties.input.properties.payload.enabled"; assertThat(source.getValue(path), is(false)); From 59fd7bf502eebebf08f56cd73b376f5c5f171851 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:50:37 +1100 Subject: [PATCH 012/204] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT test {lookup-join.LookupMessageFromIndex SYNC} #117980 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index c3fe9425def2d..871afa5555f58 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -251,6 +251,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT method: test {lookup-join.LookupMessageFromIndexKeep ASYNC} issue: https://github.com/elastic/elasticsearch/issues/117976 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {lookup-join.LookupMessageFromIndex SYNC} + issue: https://github.com/elastic/elasticsearch/issues/117980 # Examples: # From 2244070da4f34d08f6bfc20150c86129251292e4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:50:57 +1100 Subject: [PATCH 013/204] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT #117981 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 871afa5555f58..651a1ee1ad1c9 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -254,6 +254,8 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT method: test {lookup-join.LookupMessageFromIndex SYNC} issue: https://github.com/elastic/elasticsearch/issues/117980 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + issue: https://github.com/elastic/elasticsearch/issues/117981 # Examples: # From 97601b1b0b1c2249fcd55d13994981afa6eeeab3 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Wed, 4 Dec 2024 08:10:56 -0500 Subject: [PATCH 014/204] Kderusso/sparse vector ci failure (#117930) * Fix CI failure in SparseVectorQueryBuilderTests * Unmute test --- muted-tests.yml | 3 --- .../xpack/core/ml/search/SparseVectorQueryBuilderTests.java | 6 ++++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 651a1ee1ad1c9..de11f0da32c44 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -221,9 +221,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/117815 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT issue: https://github.com/elastic/elasticsearch/issues/111319 -- class: org.elasticsearch.xpack.core.ml.search.SparseVectorQueryBuilderTests - method: testToQuery - issue: https://github.com/elastic/elasticsearch/issues/117904 - class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests method: test20NoAutoGenerationWhenAutoConfigurationDisabled issue: https://github.com/elastic/elasticsearch/issues/117891 diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilderTests.java index 9872d95de024a..a5c1ba45d90b7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilderTests.java @@ -232,6 +232,12 @@ public void testToQuery() throws IOException { private void testDoToQuery(SparseVectorQueryBuilder queryBuilder, SearchExecutionContext context) throws IOException { Query query = queryBuilder.doToQuery(context); + + // test query builder can randomly have no vectors, which rewrites to a MatchNoneQuery - nothing more to do in this case. + if (query instanceof MatchNoDocsQuery) { + return; + } + assertTrue(query instanceof SparseVectorQueryWrapper); var sparseQuery = (SparseVectorQueryWrapper) query; if (queryBuilder.shouldPruneTokens()) { From ba1d5be199293d843414c5b1e4ea153ce651a06f Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:22:26 +0100 Subject: [PATCH 015/204] Updates minimum_number_of_allocations description (#117746) --- docs/reference/ml/ml-shared.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index d01047eac9815..4948db48664ed 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -18,7 +18,8 @@ end::adaptive-allocation-max-number[] tag::adaptive-allocation-min-number[] Specifies the minimum number of allocations to scale to. -If set, it must be greater than or equal to `1`. +If set, it must be greater than or equal to `0`. +If not defined, the deployment scales to `0`. end::adaptive-allocation-min-number[] tag::aggregations[] From 6855a4ecd3ea8989c995763dafcc0cdbcb25bc41 Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:22:59 +0100 Subject: [PATCH 016/204] [DOCS] Adds adaptive allocations information to Inference APIs (#117546) * Adds adaptive allocations information to Inference APIs * Update docs/reference/inference/inference-apis.asciidoc Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> * Update docs/reference/inference/put-inference.asciidoc Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> * Update docs/reference/inference/inference-apis.asciidoc Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> --------- Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> --- docs/reference/inference/inference-apis.asciidoc | 13 +++++++++++++ docs/reference/inference/put-inference.asciidoc | 15 ++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc index 037d7abeb2a36..c7b779a994a05 100644 --- a/docs/reference/inference/inference-apis.asciidoc +++ b/docs/reference/inference/inference-apis.asciidoc @@ -35,6 +35,19 @@ Elastic –, then create an {infer} endpoint by the <>. Now use <> to perform <> on your data. +[discrete] +[[adaptive-allocations]] +=== Adaptive allocations + +Adaptive allocations allow inference services to dynamically adjust the number of model allocations based on the current load. + +When adaptive allocations are enabled: + +* The number of allocations scales up automatically when the load increases. +- Allocations scale down to a minimum of 0 when the load decreases, saving resources. + +For more information about adaptive allocations and resources, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] documentation. + //[discrete] //[[default-enpoints]] //=== Default {infer} endpoints diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index e7e25ec98b49d..ed93c290b6ad4 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -67,4 +67,17 @@ Click the links to review the configuration details of the services: * <> (`text_embedding`) The {es} and ELSER services run on a {ml} node in your {es} cluster. The rest of -the services connect to external providers. \ No newline at end of file +the services connect to external providers. + +[discrete] +[[adaptive-allocations-put-inference]] +==== Adaptive allocations + +Adaptive allocations allow inference services to dynamically adjust the number of model allocations based on the current load. + +When adaptive allocations are enabled: + +- The number of allocations scales up automatically when the load increases. +- Allocations scale down to a minimum of 0 when the load decreases, saving resources. + +For more information about adaptive allocations and resources, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] documentation. \ No newline at end of file From ec27e20c80f402e09822f5e1b4ebc7b02fc6196d Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 4 Dec 2024 07:37:57 -0600 Subject: [PATCH 017/204] Removing unnecessary state from DataStreamReindexTask (#117942) --- ...indexDataStreamPersistentTaskExecutor.java | 9 ++--- .../migrate/task/ReindexDataStreamTask.java | 35 ++++++++++--------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java index e2a41ea186643..0f3f8b17f27ad 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java @@ -51,7 +51,6 @@ protected ReindexDataStreamTask createTask( params.startTime(), params.totalIndices(), params.totalIndicesToBeUpgraded(), - threadPool, id, type, action, @@ -74,9 +73,11 @@ protected void nodeOperation(AllocatedPersistentTask task, ReindexDataStreamTask List indicesToBeReindexed = indices.stream() .filter(index -> clusterService.state().getMetadata().index(index).getCreationVersion().isLegacyIndexVersion()) .toList(); - reindexDataStreamTask.setPendingIndices(indicesToBeReindexed.stream().map(Index::getName).toList()); + reindexDataStreamTask.setPendingIndicesCount(indicesToBeReindexed.size()); for (Index index : indicesToBeReindexed) { + reindexDataStreamTask.incrementInProgressIndicesCount(); // TODO This is just a placeholder. This is where the real data stream reindex logic will go + reindexDataStreamTask.reindexSucceeded(); } completeSuccessfulPersistentTask(reindexDataStreamTask); @@ -87,12 +88,12 @@ protected void nodeOperation(AllocatedPersistentTask task, ReindexDataStreamTask } private void completeSuccessfulPersistentTask(ReindexDataStreamTask persistentTask) { - persistentTask.reindexSucceeded(); + persistentTask.allReindexesCompleted(); threadPool.schedule(persistentTask::markAsCompleted, getTimeToLive(persistentTask), threadPool.generic()); } private void completeFailedPersistentTask(ReindexDataStreamTask persistentTask, Exception e) { - persistentTask.reindexFailed(e); + persistentTask.taskFailed(e); threadPool.schedule(() -> persistentTask.markAsFailed(e), getTimeToLive(persistentTask), threadPool.generic()); } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java index 722b30d9970db..72ddb87e9dea5 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java @@ -10,29 +10,27 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.persistent.AllocatedPersistentTask; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; public class ReindexDataStreamTask extends AllocatedPersistentTask { public static final String TASK_NAME = "reindex-data-stream"; private final long persistentTaskStartTime; private final int totalIndices; private final int totalIndicesToBeUpgraded; - private final ThreadPool threadPool; private boolean complete = false; private Exception exception; - private List inProgress = new ArrayList<>(); - private List pending = List.of(); + private AtomicInteger inProgress = new AtomicInteger(0); + private AtomicInteger pending = new AtomicInteger(); private List> errors = new ArrayList<>(); public ReindexDataStreamTask( long persistentTaskStartTime, int totalIndices, int totalIndicesToBeUpgraded, - ThreadPool threadPool, long id, String type, String action, @@ -44,7 +42,6 @@ public ReindexDataStreamTask( this.persistentTaskStartTime = persistentTaskStartTime; this.totalIndices = totalIndices; this.totalIndicesToBeUpgraded = totalIndicesToBeUpgraded; - this.threadPool = threadPool; } @Override @@ -55,30 +52,36 @@ public ReindexDataStreamStatus getStatus() { totalIndicesToBeUpgraded, complete, exception, - inProgress.size(), - pending.size(), + inProgress.get(), + pending.get(), errors ); } - public void reindexSucceeded() { + public void allReindexesCompleted() { this.complete = true; } - public void reindexFailed(Exception e) { + public void taskFailed(Exception e) { this.complete = true; this.exception = e; } - public void setInProgressIndices(List inProgressIndices) { - this.inProgress = inProgressIndices; + public void reindexSucceeded() { + inProgress.decrementAndGet(); + } + + public void reindexFailed(String index, Exception error) { + this.errors.add(Tuple.tuple(index, error)); + inProgress.decrementAndGet(); } - public void setPendingIndices(List pendingIndices) { - this.pending = pendingIndices; + public void incrementInProgressIndicesCount() { + inProgress.incrementAndGet(); + pending.decrementAndGet(); } - public void addErrorIndex(String index, Exception error) { - this.errors.add(Tuple.tuple(index, error)); + public void setPendingIndicesCount(int size) { + pending.set(size); } } From bbf986dd2c46a1ea4fec2eabe12b5706ab4f3682 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 4 Dec 2024 10:20:49 -0500 Subject: [PATCH 018/204] ESQL: Limit size of query (#117898) Queries bigger than a mb tend to take a lot of memory. In the worse case it's an astounding amount of memory. --- docs/changelog/117898.yaml | 5 +++++ .../xpack/esql/heap_attack/HeapAttackIT.java | 21 +++++++++++++++++++ .../xpack/esql/parser/EsqlParser.java | 17 ++++++++++++++- .../xpack/esql/analysis/ParsingTests.java | 8 +++++++ 4 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/117898.yaml diff --git a/docs/changelog/117898.yaml b/docs/changelog/117898.yaml new file mode 100644 index 0000000000000..c60061abc49ff --- /dev/null +++ b/docs/changelog/117898.yaml @@ -0,0 +1,5 @@ +pr: 117898 +summary: Limit size of query +area: ES|QL +type: bug +issues: [] diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index 8b9176a346e30..ace3db377664c 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -194,6 +194,13 @@ private void assertCircuitBreaks(ThrowingRunnable r) throws IOException { ); } + private void assertParseFailure(ThrowingRunnable r) throws IOException { + ResponseException e = expectThrows(ResponseException.class, r); + Map map = responseAsMap(e.getResponse()); + logger.info("expected parse failure {}", map); + assertMap(map, matchesMap().entry("status", 400).entry("error", matchesMap().extraOk().entry("type", "parsing_exception"))); + } + private Response sortByManyLongs(int count) throws IOException { logger.info("sorting by {} longs", count); return query(makeSortByManyLongs(count).toString(), null); @@ -318,6 +325,13 @@ public void testManyConcatFromRow() throws IOException { assertManyStrings(resp, strings); } + /** + * Fails to parse a huge huge query. + */ + public void testHugeHugeManyConcatFromRow() throws IOException { + assertParseFailure(() -> manyConcat("ROW a=9999, b=9999, c=9999, d=9999, e=9999", 50000)); + } + /** * Tests that generate many moderately long strings. */ @@ -378,6 +392,13 @@ public void testManyRepeatFromRow() throws IOException { assertManyStrings(resp, strings); } + /** + * Fails to parse a huge huge query. + */ + public void testHugeHugeManyRepeatFromRow() throws IOException { + assertParseFailure(() -> manyRepeat("ROW a = 99", 100000)); + } + /** * Tests that generate many moderately long strings. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java index 620a25e0170ea..2e55b4df1e223 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java @@ -33,6 +33,15 @@ public class EsqlParser { private static final Logger log = LogManager.getLogger(EsqlParser.class); + /** + * Maximum number of characters in an ESQL query. Antlr may parse the entire + * query into tokens to make the choices, buffering the world. There's a lot we + * can do in the grammar to prevent that, but let's be paranoid and assume we'll + * fail at preventing antlr from slurping in the world. Instead, let's make sure + * that the world just isn't that big. + */ + public static final int MAX_LENGTH = 1_000_000; + private EsqlConfig config = new EsqlConfig(); public EsqlConfig config() { @@ -60,8 +69,14 @@ private T invokeParser( Function parseFunction, BiFunction result ) { + if (query.length() > MAX_LENGTH) { + throw new org.elasticsearch.xpack.esql.core.ParsingException( + "ESQL statement is too large [{} characters > {}]", + query.length(), + MAX_LENGTH + ); + } try { - // new CaseChangingCharStream() EsqlBaseLexer lexer = new EsqlBaseLexer(CharStreams.fromString(query)); lexer.removeErrorListeners(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java index 3cafd42b731f6..68529e99c6b1b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java @@ -103,6 +103,14 @@ public void testInlineCast() throws IOException { logger.info("Wrote to file: {}", file); } + public void testTooBigQuery() { + StringBuilder query = new StringBuilder("FROM foo | EVAL a = a"); + while (query.length() < EsqlParser.MAX_LENGTH) { + query.append(", a = CONCAT(a, a)"); + } + assertEquals("-1:0: ESQL statement is too large [1000011 characters > 1000000]", error(query.toString())); + } + private String functionName(EsqlFunctionRegistry registry, Expression functionCall) { for (FunctionDefinition def : registry.listFunctions()) { if (functionCall.getClass().equals(def.clazz())) { From cb35dc96f689e1e7ed4b7ce21a9923ce6d796405 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 4 Dec 2024 07:30:26 -0800 Subject: [PATCH 019/204] Make Murmur3Hasher#update public (#117961) Closes #117883 --- .../java/org/elasticsearch/common/hash/Murmur3Hasher.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/hash/Murmur3Hasher.java b/server/src/main/java/org/elasticsearch/common/hash/Murmur3Hasher.java index 817587771d795..aec28484138fb 100644 --- a/server/src/main/java/org/elasticsearch/common/hash/Murmur3Hasher.java +++ b/server/src/main/java/org/elasticsearch/common/hash/Murmur3Hasher.java @@ -40,7 +40,12 @@ public void update(byte[] inputBytes) { update(inputBytes, 0, inputBytes.length); } - private void update(byte[] inputBytes, int offset, int length) { + /** + * Similar to {@link #update(byte[])}, but processes a specific portion of the input bytes + * starting from the given {@code offset} for the specified {@code length}. + * @see #update(byte[]) + */ + public void update(byte[] inputBytes, int offset, int length) { if (remainderLength + length >= remainder.length) { if (remainderLength > 0) { // fill rest of remainder from inputBytes and hash remainder From ef11b44b4319e0f5cfef9186519ed1f7dc2fede2 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 4 Dec 2024 07:32:00 -0800 Subject: [PATCH 020/204] Acquire stats searcher for data stream stats (#117953) Here, we only need to extract the minimum and maximum values of the timestamp field; therefore, using a stats searcher should suffice. This is important for frozen indices. --- docs/changelog/117953.yaml | 5 +++++ .../datastreams/action/DataStreamsStatsTransportAction.java | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/117953.yaml diff --git a/docs/changelog/117953.yaml b/docs/changelog/117953.yaml new file mode 100644 index 0000000000000..62f0218b1cdc7 --- /dev/null +++ b/docs/changelog/117953.yaml @@ -0,0 +1,5 @@ +pr: 117953 +summary: Acquire stats searcher for data stream stats +area: Data streams +type: bug +issues: [] diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java index 1b0b0aa6abebe..1d3b1b676282a 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.ReadOnlyEngine; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.indices.IndicesService; @@ -130,7 +131,7 @@ protected void shardOperation( DataStream dataStream = indexAbstraction.getParentDataStream(); assert dataStream != null; long maxTimestamp = 0L; - try (Engine.Searcher searcher = indexShard.acquireSearcher("data_stream_stats")) { + try (Engine.Searcher searcher = indexShard.acquireSearcher(ReadOnlyEngine.FIELD_RANGE_SEARCH_SOURCE)) { IndexReader indexReader = searcher.getIndexReader(); byte[] maxPackedValue = PointValues.getMaxPackedValue(indexReader, DataStream.TIMESTAMP_FIELD_NAME); if (maxPackedValue != null) { From 6b329907dada4ad29a4093bf00ecb71319b95cfd Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 5 Dec 2024 02:36:05 +1100 Subject: [PATCH 021/204] Mute org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests test30NoAutogenerationWhenDaemonized #117956 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index de11f0da32c44..beb8cb9591b4f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -253,6 +253,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/117980 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT issue: https://github.com/elastic/elasticsearch/issues/117981 +- class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests + method: test30NoAutogenerationWhenDaemonized + issue: https://github.com/elastic/elasticsearch/issues/117956 # Examples: # From 94f65797ccb0378779550a2d4fca2b1528d65197 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 5 Dec 2024 02:36:09 +1100 Subject: [PATCH 022/204] Mute org.elasticsearch.packaging.test.CertGenCliTests test40RunWithCert #117955 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index beb8cb9591b4f..317b960f36c56 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -256,6 +256,9 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests method: test30NoAutogenerationWhenDaemonized issue: https://github.com/elastic/elasticsearch/issues/117956 +- class: org.elasticsearch.packaging.test.CertGenCliTests + method: test40RunWithCert + issue: https://github.com/elastic/elasticsearch/issues/117955 # Examples: # From 9cf0cf155ad1069fe2e8fdaf830c3bc9a5e3d740 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 4 Dec 2024 17:05:24 +0100 Subject: [PATCH 023/204] Refactor how we build the final aggregations in GlobalOrdinalsStringTermsAggregator (#117627) This change refactor the method #buildAggregations(LongArray owningBucketOrds) so it is specific to the Collection strategy. --- .../GlobalOrdinalsStringTermsAggregator.java | 261 +++++++++--------- 1 file changed, 133 insertions(+), 128 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java index 4cf710232c7a0..0ec03a6f56dd9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java @@ -20,12 +20,10 @@ import org.apache.lucene.util.PriorityQueue; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.LongArray; import org.elasticsearch.common.util.LongHash; import org.elasticsearch.common.util.ObjectArray; import org.elasticsearch.common.util.ObjectArrayPriorityQueue; -import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.search.DocValueFormat; @@ -102,14 +100,14 @@ public GlobalOrdinalsStringTermsAggregator( this.valueCount = valuesSupplier.get().getValueCount(); this.acceptedGlobalOrdinals = acceptedOrds; if (remapGlobalOrds) { - this.collectionStrategy = new RemapGlobalOrds(cardinality, excludeDeletedDocs); + this.collectionStrategy = new RemapGlobalOrds<>(this.resultStrategy, cardinality, excludeDeletedDocs); } else { this.collectionStrategy = cardinality.map(estimate -> { if (estimate > 1) { // This is a 500 class error, because we should never be able to reach it. throw new AggregationExecutionException("Dense ords don't know how to collect from many buckets"); } - return new DenseGlobalOrds(excludeDeletedDocs); + return new DenseGlobalOrds<>(this.resultStrategy, excludeDeletedDocs); }); } } @@ -193,7 +191,13 @@ public void collect(int doc, long owningBucketOrd) throws IOException { @Override public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOException { - return resultStrategy.buildAggregations(owningBucketOrds); + if (valueCount == 0) { // no context in this reader + return GlobalOrdinalsStringTermsAggregator.this.buildAggregations( + Math.toIntExact(owningBucketOrds.size()), + ordIdx -> resultStrategy.buildNoValuesResult(owningBucketOrds.get(ordIdx)) + ); + } + return collectionStrategy.buildAggregations(owningBucketOrds); } @Override @@ -401,8 +405,8 @@ private void mapSegmentCountsToGlobalCounts(LongUnaryOperator mapping) throws IO * The {@link GlobalOrdinalsStringTermsAggregator} uses one of these * to collect the global ordinals by calling * {@link CollectionStrategy#collectGlobalOrd} for each global ordinal - * that it hits and then calling {@link CollectionStrategy#forEach} - * once to iterate on the results. + * that it hits and then calling {@link CollectionStrategy#buildAggregations} + * to generate the results. */ abstract static class CollectionStrategy implements Releasable { /** @@ -438,15 +442,9 @@ abstract static class CollectionStrategy implements Releasable { abstract long globalOrdToBucketOrd(long owningBucketOrd, long globalOrd); /** - * Iterate all of the buckets. Implementations take into account - * the {@link BucketCountThresholds}. In particular, - * if the {@link BucketCountThresholds#getMinDocCount()} is 0 then - * they'll make sure to iterate a bucket even if it was never - * {{@link #collectGlobalOrd collected}. - * If {@link BucketCountThresholds#getMinDocCount()} is not 0 then - * they'll skip all global ords that weren't collected. + * Create the aggregation result */ - abstract void forEach(long owningBucketOrd, BucketInfoConsumer consumer) throws IOException; + abstract InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOException; } interface BucketInfoConsumer { @@ -457,12 +455,17 @@ interface BucketInfoConsumer { * {@linkplain CollectionStrategy} that just uses the global ordinal as the * bucket ordinal. */ - class DenseGlobalOrds extends CollectionStrategy { + class DenseGlobalOrds< + R extends InternalAggregation, + B extends InternalMultiBucketAggregation.InternalBucket, + TB extends InternalMultiBucketAggregation.InternalBucket> extends CollectionStrategy { private final boolean excludeDeletedDocs; + private final ResultStrategy collectionStrategy; - DenseGlobalOrds(boolean excludeDeletedDocs) { + DenseGlobalOrds(ResultStrategy collectionStrategy, boolean excludeDeletedDocs) { this.excludeDeletedDocs = excludeDeletedDocs; + this.collectionStrategy = collectionStrategy; } @Override @@ -492,9 +495,7 @@ long globalOrdToBucketOrd(long owningBucketOrd, long globalOrd) { return globalOrd; } - @Override - void forEach(long owningBucketOrd, BucketInfoConsumer consumer) throws IOException { - assert owningBucketOrd == 0; + private void collect(BucketInfoConsumer consumer) throws IOException { if (excludeDeletedDocs) { forEachExcludeDeletedDocs(consumer); } else { @@ -518,7 +519,7 @@ private void forEachAllowDeletedDocs(BucketInfoConsumer consumer) throws IOExcep * Excludes deleted docs in the results by cross-checking with liveDocs. */ private void forEachExcludeDeletedDocs(BucketInfoConsumer consumer) throws IOException { - try (LongHash accepted = new LongHash(20, new BigArrays(null, null, ""))) { + try (LongHash accepted = new LongHash(20, bigArrays())) { for (LeafReaderContext ctx : searcher().getTopReaderContext().leaves()) { LeafReader reader = ctx.reader(); Bits liveDocs = reader.getLiveDocs(); @@ -550,6 +551,55 @@ private void forEachExcludeDeletedDocs(BucketInfoConsumer consumer) throws IOExc @Override public void close() {} + + @Override + InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOException { + assert owningBucketOrds.size() == 1 && owningBucketOrds.get(0) == 0; + try ( + LongArray otherDocCount = bigArrays().newLongArray(1, true); + ObjectArray topBucketsPreOrd = collectionStrategy.buildTopBucketsPerOrd(1) + ) { + GlobalOrdLookupFunction lookupGlobalOrd = valuesSupplier.get()::lookupOrd; + final int size = (int) Math.min(valueCount, bucketCountThresholds.getShardSize()); + try (ObjectArrayPriorityQueue ordered = collectionStrategy.buildPriorityQueue(size)) { + BucketUpdater updater = collectionStrategy.bucketUpdater(0, lookupGlobalOrd); + collect(new BucketInfoConsumer() { + TB spare = null; + + @Override + public void accept(long globalOrd, long bucketOrd, long docCount) throws IOException { + otherDocCount.increment(0, docCount); + if (docCount >= bucketCountThresholds.getShardMinDocCount()) { + if (spare == null) { + checkRealMemoryCBForInternalBucket(); + spare = collectionStrategy.buildEmptyTemporaryBucket(); + } + updater.updateBucket(spare, globalOrd, bucketOrd, docCount); + spare = ordered.insertWithOverflow(spare); + } + } + }); + + // Get the top buckets + topBucketsPreOrd.set(0, collectionStrategy.buildBuckets((int) ordered.size())); + for (int i = (int) ordered.size() - 1; i >= 0; --i) { + checkRealMemoryCBForInternalBucket(); + B bucket = collectionStrategy.convertTempBucketToRealBucket(ordered.pop(), lookupGlobalOrd); + topBucketsPreOrd.get(0)[i] = bucket; + otherDocCount.increment(0, -bucket.getDocCount()); + } + } + collectionStrategy.buildSubAggs(topBucketsPreOrd); + return GlobalOrdinalsStringTermsAggregator.this.buildAggregations( + Math.toIntExact(owningBucketOrds.size()), + ordIdx -> collectionStrategy.buildResult( + owningBucketOrds.get(ordIdx), + otherDocCount.get(ordIdx), + topBucketsPreOrd.get(ordIdx) + ) + ); + } + } } /** @@ -558,13 +608,22 @@ public void close() {} * {@link DenseGlobalOrds} when collecting every ordinal, but significantly * less when collecting only a few. */ - private class RemapGlobalOrds extends CollectionStrategy { + private class RemapGlobalOrds< + R extends InternalAggregation, + B extends InternalMultiBucketAggregation.InternalBucket, + TB extends InternalMultiBucketAggregation.InternalBucket> extends CollectionStrategy { private final LongKeyedBucketOrds bucketOrds; private final boolean excludeDeletedDocs; + private final ResultStrategy collectionStrategy; - private RemapGlobalOrds(CardinalityUpperBound cardinality, boolean excludeDeletedDocs) { + private RemapGlobalOrds( + ResultStrategy collectionStrategy, + CardinalityUpperBound cardinality, + boolean excludeDeletedDocs + ) { bucketOrds = LongKeyedBucketOrds.buildForValueRange(bigArrays(), cardinality, 0, valueCount - 1); this.excludeDeletedDocs = excludeDeletedDocs; + this.collectionStrategy = collectionStrategy; } @Override @@ -596,30 +655,14 @@ long globalOrdToBucketOrd(long owningBucketOrd, long globalOrd) { return bucketOrds.find(owningBucketOrd, globalOrd); } - @Override - void forEach(long owningBucketOrd, BucketInfoConsumer consumer) throws IOException { + private void collectZeroDocEntriesIfNeeded(long owningBucketOrd) throws IOException { if (excludeDeletedDocs) { - forEachExcludeDeletedDocs(owningBucketOrd, consumer); - } else { - forEachAllowDeletedDocs(owningBucketOrd, consumer); - } - } - - void forEachAllowDeletedDocs(long owningBucketOrd, BucketInfoConsumer consumer) throws IOException { - if (bucketCountThresholds.getMinDocCount() == 0) { + forEachExcludeDeletedDocs(owningBucketOrd); + } else if (bucketCountThresholds.getMinDocCount() == 0) { for (long globalOrd = 0; globalOrd < valueCount; globalOrd++) { - if (false == acceptedGlobalOrdinals.test(globalOrd)) { - continue; - } - addBucketForMinDocCountZero(owningBucketOrd, globalOrd, consumer, null); - } - } else { - LongKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrd); - while (ordsEnum.next()) { - if (false == acceptedGlobalOrdinals.test(ordsEnum.value())) { - continue; + if (acceptedGlobalOrdinals.test(globalOrd)) { + bucketOrds.add(owningBucketOrd, globalOrd); } - consumer.accept(ordsEnum.value(), ordsEnum.ord(), bucketDocCount(ordsEnum.ord())); } } } @@ -627,9 +670,9 @@ void forEachAllowDeletedDocs(long owningBucketOrd, BucketInfoConsumer consumer) /** * Excludes deleted docs in the results by cross-checking with liveDocs. */ - void forEachExcludeDeletedDocs(long owningBucketOrd, BucketInfoConsumer consumer) throws IOException { + private void forEachExcludeDeletedDocs(long owningBucketOrd) throws IOException { assert bucketCountThresholds.getMinDocCount() == 0; - try (LongHash accepted = new LongHash(20, new BigArrays(null, null, ""))) { + try (LongHash accepted = new LongHash(20, bigArrays())) { for (LeafReaderContext ctx : searcher().getTopReaderContext().leaves()) { LeafReader reader = ctx.reader(); Bits liveDocs = reader.getLiveDocs(); @@ -646,7 +689,8 @@ void forEachExcludeDeletedDocs(long owningBucketOrd, BucketInfoConsumer consumer if (false == acceptedGlobalOrdinals.test(globalOrd)) { continue; } - addBucketForMinDocCountZero(owningBucketOrd, globalOrd, consumer, accepted); + bucketOrds.add(owningBucketOrd, globalOrd); + accepted.add(globalOrd); } } } @@ -655,110 +699,71 @@ void forEachExcludeDeletedDocs(long owningBucketOrd, BucketInfoConsumer consumer } } - private void addBucketForMinDocCountZero( - long owningBucketOrd, - long globalOrd, - BucketInfoConsumer consumer, - @Nullable LongHash accepted - ) throws IOException { - /* - * Use `add` instead of `find` here to assign an ordinal - * even if the global ord wasn't found so we can build - * sub-aggregations without trouble even though we haven't - * hit any documents for them. This is wasteful, but - * settings minDocCount == 0 is wasteful in general..... - */ - long bucketOrd = bucketOrds.add(owningBucketOrd, globalOrd); - long docCount; - if (bucketOrd < 0) { - bucketOrd = -1 - bucketOrd; - docCount = bucketDocCount(bucketOrd); - } else { - docCount = 0; - } - assert globalOrd >= 0; - consumer.accept(globalOrd, bucketOrd, docCount); - if (accepted != null) { - accepted.add(globalOrd); - } - } - @Override public void close() { bucketOrds.close(); } - } - - /** - * Strategy for building results. - */ - abstract class ResultStrategy< - R extends InternalAggregation, - B extends InternalMultiBucketAggregation.InternalBucket, - TB extends InternalMultiBucketAggregation.InternalBucket> implements Releasable { - - private InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOException { - if (valueCount == 0) { // no context in this reader - return GlobalOrdinalsStringTermsAggregator.this.buildAggregations( - Math.toIntExact(owningBucketOrds.size()), - ordIdx -> buildNoValuesResult(owningBucketOrds.get(ordIdx)) - ); - } + @Override + InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOException { try ( LongArray otherDocCount = bigArrays().newLongArray(owningBucketOrds.size(), true); - ObjectArray topBucketsPreOrd = buildTopBucketsPerOrd(owningBucketOrds.size()) + ObjectArray topBucketsPreOrd = collectionStrategy.buildTopBucketsPerOrd(owningBucketOrds.size()) ) { GlobalOrdLookupFunction lookupGlobalOrd = valuesSupplier.get()::lookupOrd; for (long ordIdx = 0; ordIdx < topBucketsPreOrd.size(); ordIdx++) { - final int size; - if (bucketCountThresholds.getMinDocCount() == 0) { - // if minDocCount == 0 then we can end up with more buckets then maxBucketOrd() returns - size = (int) Math.min(valueCount, bucketCountThresholds.getShardSize()); - } else { - size = (int) Math.min(maxBucketOrd(), bucketCountThresholds.getShardSize()); - } - try (ObjectArrayPriorityQueue ordered = buildPriorityQueue(size)) { - final long finalOrdIdx = ordIdx; - final long owningBucketOrd = owningBucketOrds.get(ordIdx); - BucketUpdater updater = bucketUpdater(owningBucketOrd, lookupGlobalOrd); - collectionStrategy.forEach(owningBucketOrd, new BucketInfoConsumer() { - TB spare = null; - - @Override - public void accept(long globalOrd, long bucketOrd, long docCount) throws IOException { - otherDocCount.increment(finalOrdIdx, docCount); - if (docCount >= bucketCountThresholds.getShardMinDocCount()) { - if (spare == null) { - checkRealMemoryCBForInternalBucket(); - spare = buildEmptyTemporaryBucket(); - } - updater.updateBucket(spare, globalOrd, bucketOrd, docCount); - spare = ordered.insertWithOverflow(spare); - } + long owningBucketOrd = owningBucketOrds.get(ordIdx); + collectZeroDocEntriesIfNeeded(owningBucketOrds.get(ordIdx)); + int size = (int) Math.min(bucketOrds.bucketsInOrd(owningBucketOrd), bucketCountThresholds.getShardSize()); + try (ObjectArrayPriorityQueue ordered = collectionStrategy.buildPriorityQueue(size)) { + BucketUpdater updater = collectionStrategy.bucketUpdater(owningBucketOrd, lookupGlobalOrd); + LongKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrd); + TB spare = null; + while (ordsEnum.next()) { + long docCount = bucketDocCount(ordsEnum.ord()); + otherDocCount.increment(ordIdx, docCount); + if (docCount < bucketCountThresholds.getShardMinDocCount()) { + continue; } - }); - + if (spare == null) { + checkRealMemoryCBForInternalBucket(); + spare = collectionStrategy.buildEmptyTemporaryBucket(); + } + updater.updateBucket(spare, ordsEnum.value(), ordsEnum.ord(), docCount); + spare = ordered.insertWithOverflow(spare); + } // Get the top buckets - topBucketsPreOrd.set(ordIdx, buildBuckets((int) ordered.size())); + topBucketsPreOrd.set(ordIdx, collectionStrategy.buildBuckets((int) ordered.size())); for (int i = (int) ordered.size() - 1; i >= 0; --i) { checkRealMemoryCBForInternalBucket(); - B bucket = convertTempBucketToRealBucket(ordered.pop(), lookupGlobalOrd); + B bucket = collectionStrategy.convertTempBucketToRealBucket(ordered.pop(), lookupGlobalOrd); topBucketsPreOrd.get(ordIdx)[i] = bucket; otherDocCount.increment(ordIdx, -bucket.getDocCount()); } } } - - buildSubAggs(topBucketsPreOrd); - + collectionStrategy.buildSubAggs(topBucketsPreOrd); return GlobalOrdinalsStringTermsAggregator.this.buildAggregations( Math.toIntExact(owningBucketOrds.size()), - ordIdx -> buildResult(owningBucketOrds.get(ordIdx), otherDocCount.get(ordIdx), topBucketsPreOrd.get(ordIdx)) + ordIdx -> collectionStrategy.buildResult( + owningBucketOrds.get(ordIdx), + otherDocCount.get(ordIdx), + topBucketsPreOrd.get(ordIdx) + ) ); } } + } + + /** + * Strategy for building results. + */ + abstract class ResultStrategy< + R extends InternalAggregation, + B extends InternalMultiBucketAggregation.InternalBucket, + TB extends InternalMultiBucketAggregation.InternalBucket> implements Releasable { + /** * Short description of the collection mechanism added to the profile * output to help with debugging. @@ -780,7 +785,7 @@ public void accept(long globalOrd, long bucketOrd, long docCount) throws IOExcep * Update fields in {@code spare} to reflect information collected for * this bucket ordinal. */ - abstract BucketUpdater bucketUpdater(long owningBucketOrd, GlobalOrdLookupFunction lookupGlobalOrd) throws IOException; + abstract BucketUpdater bucketUpdater(long owningBucketOrd, GlobalOrdLookupFunction lookupGlobalOrd); /** * Build a {@link PriorityQueue} to sort the buckets. After we've @@ -862,7 +867,7 @@ OrdBucket buildEmptyTemporaryBucket() { } @Override - BucketUpdater bucketUpdater(long owningBucketOrd, GlobalOrdLookupFunction lookupGlobalOrd) throws IOException { + BucketUpdater bucketUpdater(long owningBucketOrd, GlobalOrdLookupFunction lookupGlobalOrd) { return (spare, globalOrd, bucketOrd, docCount) -> { spare.globalOrd = globalOrd; spare.bucketOrd = bucketOrd; From 0901a2734ee7dad7da84b9590eb942f09b4c5952 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 4 Dec 2024 17:29:46 +0000 Subject: [PATCH 024/204] Add option to store `sparse_vector` outside `_source` (#117917) This PR introduces an option for `sparse_vector` to store its values separately from `_source` by using term vectors. This capability is primarly needed by the semantic text field. --- docs/changelog/117917.yaml | 5 + .../mapping/types/sparse-vector.asciidoc | 17 ++ .../test/search.vectors/90_sparse_vector.yml | 117 ++++++++++++ .../index/mapper/MapperFeatures.java | 4 +- .../vectors/SparseVectorFieldMapper.java | 155 ++++++++++++++- .../index/mapper/vectors/XFeatureField.java | 177 ++++++++++++++++++ .../vectors/SparseVectorFieldMapperTests.java | 135 +++++++++++-- .../vectors/SparseVectorFieldTypeTests.java | 4 +- .../mapper/SemanticTextFieldMapperTests.java | 4 +- 9 files changed, 589 insertions(+), 29 deletions(-) create mode 100644 docs/changelog/117917.yaml create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/vectors/XFeatureField.java diff --git a/docs/changelog/117917.yaml b/docs/changelog/117917.yaml new file mode 100644 index 0000000000000..b6dc90f6b903d --- /dev/null +++ b/docs/changelog/117917.yaml @@ -0,0 +1,5 @@ +pr: 117917 +summary: Add option to store `sparse_vector` outside `_source` +area: Mapping +type: feature +issues: [] diff --git a/docs/reference/mapping/types/sparse-vector.asciidoc b/docs/reference/mapping/types/sparse-vector.asciidoc index b24f65fcf97ca..22d4644ede490 100644 --- a/docs/reference/mapping/types/sparse-vector.asciidoc +++ b/docs/reference/mapping/types/sparse-vector.asciidoc @@ -26,6 +26,23 @@ PUT my-index See <> for a complete example on adding documents to a `sparse_vector` mapped field using ELSER. +[[sparse-vectors-params]] +==== Parameters for `sparse_vector` fields + +The following parameters are accepted by `sparse_vector` fields: + +[horizontal] + +<>:: + +Indicates whether the field value should be stored and retrievable independently of the <> field. +Accepted values: true or false (default). +The field's data is stored using term vectors, a disk-efficient structure compared to the original JSON input. +The input map can be retrieved during a search request via the <>. +To benefit from reduced disk usage, you must either: + * Exclude the field from <>. + * Use <>. + [[index-multi-value-sparse-vectors]] ==== Multi-value sparse vectors diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml index 2505e6d7e353b..0b65a69bf500e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml @@ -472,3 +472,120 @@ - match: _source.ml.tokens: {} + +--- +"stored sparse_vector": + + - requires: + cluster_features: [ "mapper.sparse_vector.store_support" ] + reason: "sparse_vector supports store parameter" + + - do: + indices.create: + index: test + body: + mappings: + properties: + ml.tokens: + type: sparse_vector + store: true + + - match: { acknowledged: true } + - do: + index: + index: test + id: "1" + body: + ml: + tokens: + running: 2 + good: 3 + run: 5 + race: 7 + for: 9 + + - match: { result: "created" } + + - do: + indices.refresh: { } + + - do: + search: + index: test + body: + fields: [ "ml.tokens" ] + + - length: { hits.hits.0.fields.ml\\.tokens: 1 } + - length: { hits.hits.0.fields.ml\\.tokens.0: 5 } + - match: { hits.hits.0.fields.ml\\.tokens.0.running: 2.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.good: 3.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.run: 5.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.race: 7.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.for: 9.0 } + +--- +"stored sparse_vector synthetic source": + + - requires: + cluster_features: [ "mapper.source.mode_from_index_setting", "mapper.sparse_vector.store_support" ] + reason: "sparse_vector supports store parameter" + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + ml.tokens: + type: sparse_vector + store: true + + - match: { acknowledged: true } + + - do: + index: + index: test + id: "1" + body: + ml: + tokens: + running: 2 + good: 3 + run: 5 + race: 7 + for: 9 + + - match: { result: "created" } + + - do: + indices.refresh: { } + + - do: + search: + index: test + body: + fields: [ "ml.tokens" ] + + - match: + hits.hits.0._source: { + ml: { + tokens: { + running: 2.0, + good: 3.0, + run: 5.0, + race: 7.0, + for: 9.0 + } + } + } + + - length: { hits.hits.0.fields.ml\\.tokens: 1 } + - length: { hits.hits.0.fields.ml\\.tokens.0: 5 } + - match: { hits.hits.0.fields.ml\\.tokens.0.running: 2.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.good: 3.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.run: 5.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.race: 7.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.for: 9.0 } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index ffb38d229078e..276d3e151361c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -62,6 +62,7 @@ public Set getFeatures() { ); public static final NodeFeature META_FETCH_FIELDS_ERROR_CODE_CHANGED = new NodeFeature("meta_fetch_fields_error_code_changed"); + public static final NodeFeature SPARSE_VECTOR_STORE_SUPPORT = new NodeFeature("mapper.sparse_vector.store_support"); @Override public Set getTestFeatures() { @@ -75,7 +76,8 @@ public Set getTestFeatures() { MapperService.LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT, DocumentParser.FIX_PARSING_SUBOBJECTS_FALSE_DYNAMIC_FALSE, CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX, - META_FETCH_FIELDS_ERROR_CODE_CHANGED + META_FETCH_FIELDS_ERROR_CODE_CHANGED, + SPARSE_VECTOR_STORE_SUPPORT ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java index d0a8dfae4f242..552e66336005d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java @@ -11,6 +11,12 @@ import org.apache.lucene.document.FeatureField; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.TermVectors; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; @@ -25,14 +31,22 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser.Token; import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Stream; import static org.elasticsearch.index.query.AbstractQueryBuilder.DEFAULT_BOOST; @@ -52,8 +66,12 @@ public class SparseVectorFieldMapper extends FieldMapper { static final IndexVersion NEW_SPARSE_VECTOR_INDEX_VERSION = IndexVersions.NEW_SPARSE_VECTOR; static final IndexVersion SPARSE_VECTOR_IN_FIELD_NAMES_INDEX_VERSION = IndexVersions.SPARSE_VECTOR_IN_FIELD_NAMES_SUPPORT; - public static class Builder extends FieldMapper.Builder { + private static SparseVectorFieldMapper toType(FieldMapper in) { + return (SparseVectorFieldMapper) in; + } + public static class Builder extends FieldMapper.Builder { + private final Parameter stored = Parameter.storeParam(m -> toType(m).fieldType().isStored(), false); private final Parameter> meta = Parameter.metaParam(); public Builder(String name) { @@ -62,14 +80,14 @@ public Builder(String name) { @Override protected Parameter[] getParameters() { - return new Parameter[] { meta }; + return new Parameter[] { stored, meta }; } @Override public SparseVectorFieldMapper build(MapperBuilderContext context) { return new SparseVectorFieldMapper( leafName(), - new SparseVectorFieldType(context.buildFullName(leafName()), meta.getValue()), + new SparseVectorFieldType(context.buildFullName(leafName()), stored.getValue(), meta.getValue()), builderParams(this, context) ); } @@ -87,8 +105,8 @@ public SparseVectorFieldMapper build(MapperBuilderContext context) { public static final class SparseVectorFieldType extends MappedFieldType { - public SparseVectorFieldType(String name, Map meta) { - super(name, true, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + public SparseVectorFieldType(String name, boolean isStored, Map meta) { + super(name, true, isStored, false, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); } @Override @@ -103,6 +121,9 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + if (isStored()) { + return new SparseVectorValueFetcher(name()); + } return SourceValueFetcher.identity(name(), context, format); } @@ -135,6 +156,14 @@ private SparseVectorFieldMapper(String simpleName, MappedFieldType mappedFieldTy super(simpleName, mappedFieldType, builderParams); } + @Override + protected SyntheticSourceSupport syntheticSourceSupport() { + if (fieldType().isStored()) { + return new SyntheticSourceSupport.Native(new SparseVectorSyntheticFieldLoader(fullPath(), leafName())); + } + return super.syntheticSourceSupport(); + } + @Override public Map indexAnalyzers() { return Map.of(mappedFieldType.name(), Lucene.KEYWORD_ANALYZER); @@ -189,9 +218,9 @@ public void parse(DocumentParserContext context) throws IOException { // based on recommendations from this paper: https://arxiv.org/pdf/2305.18494.pdf IndexableField currentField = context.doc().getByKey(key); if (currentField == null) { - context.doc().addWithKey(key, new FeatureField(fullPath(), feature, value)); - } else if (currentField instanceof FeatureField && ((FeatureField) currentField).getFeatureValue() < value) { - ((FeatureField) currentField).setFeatureValue(value); + context.doc().addWithKey(key, new XFeatureField(fullPath(), feature, value, fieldType().isStored())); + } else if (currentField instanceof XFeatureField && ((XFeatureField) currentField).getFeatureValue() < value) { + ((XFeatureField) currentField).setFeatureValue(value); } } else { throw new IllegalArgumentException( @@ -219,4 +248,114 @@ protected String contentType() { return CONTENT_TYPE; } + private static class SparseVectorValueFetcher implements ValueFetcher { + private final String fieldName; + private TermVectors termVectors; + + private SparseVectorValueFetcher(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public void setNextReader(LeafReaderContext context) { + try { + termVectors = context.reader().termVectors(); + } catch (IOException exc) { + throw new UncheckedIOException(exc); + } + } + + @Override + public List fetchValues(Source source, int doc, List ignoredValues) throws IOException { + if (termVectors == null) { + return List.of(); + } + var terms = termVectors.get(doc, fieldName); + if (terms == null) { + return List.of(); + } + + var termsEnum = terms.iterator(); + PostingsEnum postingsScratch = null; + Map result = new LinkedHashMap<>(); + while (termsEnum.next() != null) { + postingsScratch = termsEnum.postings(postingsScratch); + postingsScratch.nextDoc(); + result.put(termsEnum.term().utf8ToString(), XFeatureField.decodeFeatureValue(postingsScratch.freq())); + assert postingsScratch.nextDoc() == DocIdSetIterator.NO_MORE_DOCS; + } + return List.of(result); + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + } + + private static class SparseVectorSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader { + private final String fullPath; + private final String leafName; + + private TermsEnum termsDocEnum; + + private SparseVectorSyntheticFieldLoader(String fullPath, String leafName) { + this.fullPath = fullPath; + this.leafName = leafName; + } + + @Override + public Stream> storedFieldLoaders() { + return Stream.of(); + } + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + var fieldInfos = leafReader.getFieldInfos().fieldInfo(fullPath); + if (fieldInfos == null || fieldInfos.hasTermVectors() == false) { + return null; + } + return docId -> { + var terms = leafReader.termVectors().get(docId, fullPath); + if (terms == null) { + return false; + } + termsDocEnum = terms.iterator(); + if (termsDocEnum.next() == null) { + termsDocEnum = null; + return false; + } + return true; + }; + } + + @Override + public boolean hasValue() { + return termsDocEnum != null; + } + + @Override + public void write(XContentBuilder b) throws IOException { + assert termsDocEnum != null; + PostingsEnum reuse = null; + b.startObject(leafName); + do { + reuse = termsDocEnum.postings(reuse); + reuse.nextDoc(); + b.field(termsDocEnum.term().utf8ToString(), XFeatureField.decodeFeatureValue(reuse.freq())); + } while (termsDocEnum.next() != null); + b.endObject(); + } + + @Override + public String fieldName() { + return leafName; + } + + @Override + public void reset() { + termsDocEnum = null; + } + } + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/XFeatureField.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/XFeatureField.java new file mode 100644 index 0000000000000..5f4afb4a86acc --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/XFeatureField.java @@ -0,0 +1,177 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.elasticsearch.index.mapper.vectors; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute; +import org.apache.lucene.document.FeatureField; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; + +/** + * This class is forked from the Lucene {@link FeatureField} implementation to enable support for storing term vectors. + * It should be removed once apache/lucene#14034 becomes available. + */ +public final class XFeatureField extends Field { + private static final FieldType FIELD_TYPE = new FieldType(); + private static final FieldType FIELD_TYPE_STORE_TERM_VECTORS = new FieldType(); + + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + + FIELD_TYPE_STORE_TERM_VECTORS.setTokenized(false); + FIELD_TYPE_STORE_TERM_VECTORS.setOmitNorms(true); + FIELD_TYPE_STORE_TERM_VECTORS.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + FIELD_TYPE_STORE_TERM_VECTORS.setStoreTermVectors(true); + } + + private float featureValue; + + /** + * Create a feature. + * + * @param fieldName The name of the field to store the information into. All features may be + * stored in the same field. + * @param featureName The name of the feature, eg. 'pagerank`. It will be indexed as a term. + * @param featureValue The value of the feature, must be a positive, finite, normal float. + */ + public XFeatureField(String fieldName, String featureName, float featureValue) { + this(fieldName, featureName, featureValue, false); + } + + /** + * Create a feature. + * + * @param fieldName The name of the field to store the information into. All features may be + * stored in the same field. + * @param featureName The name of the feature, eg. 'pagerank`. It will be indexed as a term. + * @param featureValue The value of the feature, must be a positive, finite, normal float. + */ + public XFeatureField(String fieldName, String featureName, float featureValue, boolean storeTermVectors) { + super(fieldName, featureName, storeTermVectors ? FIELD_TYPE_STORE_TERM_VECTORS : FIELD_TYPE); + setFeatureValue(featureValue); + } + + /** + * Update the feature value of this field. + */ + public void setFeatureValue(float featureValue) { + if (Float.isFinite(featureValue) == false) { + throw new IllegalArgumentException( + "featureValue must be finite, got: " + featureValue + " for feature " + fieldsData + " on field " + name + ); + } + if (featureValue < Float.MIN_NORMAL) { + throw new IllegalArgumentException( + "featureValue must be a positive normal float, got: " + + featureValue + + " for feature " + + fieldsData + + " on field " + + name + + " which is less than the minimum positive normal float: " + + Float.MIN_NORMAL + ); + } + this.featureValue = featureValue; + } + + @Override + public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) { + FeatureTokenStream stream; + if (reuse instanceof FeatureTokenStream) { + stream = (FeatureTokenStream) reuse; + } else { + stream = new FeatureTokenStream(); + } + + int freqBits = Float.floatToIntBits(featureValue); + stream.setValues((String) fieldsData, freqBits >>> 15); + return stream; + } + + /** + * This is useful if you have multiple features sharing a name and you want to take action to + * deduplicate them. + * + * @return the feature value of this field. + */ + public float getFeatureValue() { + return featureValue; + } + + private static final class FeatureTokenStream extends TokenStream { + private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class); + private final TermFrequencyAttribute freqAttribute = addAttribute(TermFrequencyAttribute.class); + private boolean used = true; + private String value = null; + private int freq = 0; + + private FeatureTokenStream() {} + + /** + * Sets the values + */ + void setValues(String value, int freq) { + this.value = value; + this.freq = freq; + } + + @Override + public boolean incrementToken() { + if (used) { + return false; + } + clearAttributes(); + termAttribute.append(value); + freqAttribute.setTermFrequency(freq); + used = true; + return true; + } + + @Override + public void reset() { + used = false; + } + + @Override + public void close() { + value = null; + } + } + + static final int MAX_FREQ = Float.floatToIntBits(Float.MAX_VALUE) >>> 15; + + static float decodeFeatureValue(float freq) { + if (freq > MAX_FREQ) { + // This is never used in practice but callers of the SimScorer API might + // occasionally call it on eg. Float.MAX_VALUE to compute the max score + // so we need to be consistent. + return Float.MAX_VALUE; + } + int tf = (int) freq; // lossless + int featureBits = tf << 15; + return Float.intBitsToFloat(featureBits); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java index ffa5bd339ae06..8e0cd97e518fa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java @@ -11,17 +11,24 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute; -import org.apache.lucene.document.FeatureField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.search.lookup.Source; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.hamcrest.Matchers; @@ -29,18 +36,25 @@ import java.io.IOException; import java.util.Arrays; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.NEW_SPARSE_VECTOR_INDEX_VERSION; import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.PREVIOUS_SPARSE_VECTOR_INDEX_VERSION; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; public class SparseVectorFieldMapperTests extends MapperTestCase { @Override protected Object getSampleValueForDocument() { - return Map.of("ten", 10, "twenty", 20); + Map map = new LinkedHashMap<>(); + map.put("ten", 10f); + map.put("twenty", 20f); + return map; } @Override @@ -88,14 +102,18 @@ public void testDefaults() throws Exception { List fields = doc1.rootDoc().getFields("field"); assertEquals(2, fields.size()); - assertThat(fields.get(0), Matchers.instanceOf(FeatureField.class)); - FeatureField featureField1 = null; - FeatureField featureField2 = null; + if (IndexVersion.current().luceneVersion().major == 10) { + // TODO: Update to use Lucene's FeatureField after upgrading to Lucene 10.1. + assertThat(IndexVersion.current().luceneVersion().minor, equalTo(0)); + } + assertThat(fields.get(0), Matchers.instanceOf(XFeatureField.class)); + XFeatureField featureField1 = null; + XFeatureField featureField2 = null; for (IndexableField field : fields) { if (field.stringValue().equals("ten")) { - featureField1 = (FeatureField) field; + featureField1 = (XFeatureField) field; } else if (field.stringValue().equals("twenty")) { - featureField2 = (FeatureField) field; + featureField2 = (XFeatureField) field; } else { throw new UnsupportedOperationException(); } @@ -112,14 +130,14 @@ public void testDotInFieldName() throws Exception { List fields = parsedDocument.rootDoc().getFields("field"); assertEquals(2, fields.size()); - assertThat(fields.get(0), Matchers.instanceOf(FeatureField.class)); - FeatureField featureField1 = null; - FeatureField featureField2 = null; + assertThat(fields.get(0), Matchers.instanceOf(XFeatureField.class)); + XFeatureField featureField1 = null; + XFeatureField featureField2 = null; for (IndexableField field : fields) { if (field.stringValue().equals("foo.bar")) { - featureField1 = (FeatureField) field; + featureField1 = (XFeatureField) field; } else if (field.stringValue().equals("foobar")) { - featureField2 = (FeatureField) field; + featureField2 = (XFeatureField) field; } else { throw new UnsupportedOperationException(); } @@ -167,13 +185,13 @@ public void testHandlesMultiValuedFields() throws MapperParsingException, IOExce })); // then validate that the generate document stored both values appropriately and we have only the max value stored - FeatureField barField = ((FeatureField) doc1.rootDoc().getByKey("foo.field\\.bar")); + XFeatureField barField = ((XFeatureField) doc1.rootDoc().getByKey("foo.field\\.bar")); assertEquals(20, barField.getFeatureValue(), 1); - FeatureField storedBarField = ((FeatureField) doc1.rootDoc().getFields("foo.field").get(1)); + XFeatureField storedBarField = ((XFeatureField) doc1.rootDoc().getFields("foo.field").get(1)); assertEquals(20, storedBarField.getFeatureValue(), 1); - assertEquals(3, doc1.rootDoc().getFields().stream().filter((f) -> f instanceof FeatureField).count()); + assertEquals(3, doc1.rootDoc().getFields().stream().filter((f) -> f instanceof XFeatureField).count()); } public void testCannotBeUsedInMultiFields() { @@ -188,6 +206,53 @@ public void testCannotBeUsedInMultiFields() { assertThat(e.getMessage(), containsString("Field [feature] of type [sparse_vector] can't be used in multifields")); } + public void testStoreIsNotUpdateable() throws IOException { + var mapperService = createMapperService(fieldMapping(this::minimalMapping)); + XContentBuilder mapping = jsonBuilder().startObject() + .startObject("_doc") + .startObject("properties") + .startObject("field") + .field("type", "sparse_vector") + .field("store", true) + .endObject() + .endObject() + .endObject() + .endObject(); + var exc = expectThrows( + Exception.class, + () -> mapperService.merge("_doc", new CompressedXContent(Strings.toString(mapping)), MapperService.MergeReason.MAPPING_UPDATE) + ); + assertThat(exc.getMessage(), containsString("Cannot update parameter [store]")); + } + + @SuppressWarnings("unchecked") + public void testValueFetcher() throws Exception { + for (boolean store : new boolean[] { true, false }) { + var mapperService = createMapperService(fieldMapping(store ? this::minimalStoreMapping : this::minimalMapping)); + var mapper = mapperService.documentMapper(); + try (Directory directory = newDirectory()) { + RandomIndexWriter iw = new RandomIndexWriter(random(), directory); + var sourceToParse = source(this::writeField); + ParsedDocument doc1 = mapper.parse(sourceToParse); + iw.addDocument(doc1.rootDoc()); + iw.close(); + try (DirectoryReader reader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) { + LeafReader leafReader = getOnlyLeafReader(reader); + var searchContext = createSearchExecutionContext(mapperService, new IndexSearcher(leafReader)); + var fieldType = mapper.mappers().getFieldType("field"); + var valueFetcher = fieldType.valueFetcher(searchContext, null); + valueFetcher.setNextReader(leafReader.getContext()); + + var source = Source.fromBytes(sourceToParse.source()); + var result = valueFetcher.fetchValues(source, 0, List.of()); + assertThat(result.size(), equalTo(1)); + assertThat(result.get(0), instanceOf(Map.class)); + assertThat(toFloats((Map) result.get(0)), equalTo(toFloats((Map) source.source().get("field")))); + } + } + } + } + @Override protected Object generateRandomInputValue(MappedFieldType ft) { assumeFalse("Test implemented in a follow up", true); @@ -201,7 +266,29 @@ protected boolean allowsNullValues() { @Override protected SyntheticSourceSupport syntheticSourceSupport(boolean syntheticSource) { - throw new AssumptionViolatedException("not supported"); + boolean withStore = randomBoolean(); + return new SyntheticSourceSupport() { + @Override + public boolean preservesExactSource() { + return withStore == false; + } + + @Override + public SyntheticSourceExample example(int maxValues) { + return new SyntheticSourceExample(getSampleValueForDocument(), getSampleValueForDocument(), b -> { + if (withStore) { + minimalStoreMapping(b); + } else { + minimalMapping(b); + } + }); + } + + @Override + public List invalidExample() { + return List.of(); + } + }; } @Override @@ -234,4 +321,20 @@ public void testSparseVectorUnsupportedIndex() throws Exception { }))); assertThat(e.getMessage(), containsString(SparseVectorFieldMapper.ERROR_MESSAGE_8X)); } + + /** + * Handles float/double conversion when reading/writing with xcontent by converting all numbers to floats. + */ + private Map toFloats(Map value) { + // preserve order + Map result = new LinkedHashMap<>(); + for (var entry : value.entrySet()) { + if (entry.getValue() instanceof Number num) { + result.put(entry.getKey(), num.floatValue()); + } else { + throw new IllegalArgumentException("Expected Number, got: " + value.getClass().getSimpleName()); + } + } + return result; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java index 4627d4d871957..0dbe3817c3e87 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java @@ -18,13 +18,13 @@ public class SparseVectorFieldTypeTests extends FieldTypeTestCase { public void testDocValuesDisabled() { - MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType("field", Collections.emptyMap()); + MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType("field", false, Collections.emptyMap()); assertFalse(fieldType.hasDocValues()); expectThrows(IllegalArgumentException.class, () -> fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test"))); } public void testIsNotAggregatable() { - MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType("field", Collections.emptyMap()); + MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType("field", false, Collections.emptyMap()); assertFalse(fieldType.isAggregatable()); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 71ff9fc7d84cf..fd60d9687f437 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.mapper; -import org.apache.lucene.document.FeatureField; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexableField; @@ -47,6 +46,7 @@ import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; +import org.elasticsearch.index.mapper.vectors.XFeatureField; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.search.ESToParentBlockJoinQuery; import org.elasticsearch.inference.Model; @@ -1130,7 +1130,7 @@ private static void assertChildLeafNestedDocument( private static void assertSparseFeatures(LuceneDocument doc, String fieldName, int expectedCount) { int count = 0; for (IndexableField field : doc.getFields()) { - if (field instanceof FeatureField featureField) { + if (field instanceof XFeatureField featureField) { assertThat(featureField.name(), equalTo(fieldName)); ++count; } From 4efe696b1fe12d003816b4a42ed987e14d132819 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Wed, 4 Dec 2024 18:45:32 +0100 Subject: [PATCH 025/204] ES|QL: fix lookup-join csv tests with nondeterministic order (#118008) --- muted-tests.yml | 14 -------------- .../src/main/resources/lookup-join.csv-spec | 4 ++++ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 317b960f36c56..e69588d6ec359 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -239,20 +239,6 @@ tests: - class: org.elasticsearch.xpack.ml.integration.RegressionIT method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet issue: https://github.com/elastic/elasticsearch/issues/117805 -- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT - method: test {lookup-join.LookupIPFromIndexKeep ASYNC} - issue: https://github.com/elastic/elasticsearch/issues/117974 -- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT - method: test {lookup-join.LookupMessageFromIndex ASYNC} - issue: https://github.com/elastic/elasticsearch/issues/117975 -- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT - method: test {lookup-join.LookupMessageFromIndexKeep ASYNC} - issue: https://github.com/elastic/elasticsearch/issues/117976 -- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT - method: test {lookup-join.LookupMessageFromIndex SYNC} - issue: https://github.com/elastic/elasticsearch/issues/117980 -- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT - issue: https://github.com/elastic/elasticsearch/issues/117981 - class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests method: test30NoAutogenerationWhenDaemonized issue: https://github.com/elastic/elasticsearch/issues/117956 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index f2800456ceb33..712cadf6d44fd 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -127,6 +127,7 @@ FROM sample_data | EVAL client_ip = client_ip::keyword | LOOKUP JOIN clientips_lookup ON client_ip ; +ignoreOrder:true; @timestamp:date | event_duration:long | message:keyword | client_ip:keyword | env:keyword 2023-10-23T13:55:01.543Z | 1756467 | Connected to 10.1.0.1 | 172.21.3.15 | Production @@ -146,6 +147,7 @@ FROM sample_data | LOOKUP JOIN clientips_lookup ON client_ip | KEEP @timestamp, client_ip, event_duration, message, env ; +ignoreOrder:true; @timestamp:date | client_ip:keyword | event_duration:long | message:keyword | env:keyword 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Production @@ -230,6 +232,7 @@ required_capability: join_lookup_v4 FROM sample_data | LOOKUP JOIN message_types_lookup ON message ; +ignoreOrder:true; @timestamp:date | client_ip:ip | event_duration:long | message:keyword | type:keyword 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Success @@ -248,6 +251,7 @@ FROM sample_data | LOOKUP JOIN message_types_lookup ON message | KEEP @timestamp, client_ip, event_duration, message, type ; +ignoreOrder:true; @timestamp:date | client_ip:ip | event_duration:long | message:keyword | type:keyword 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Success From 774c6ea174bdd866ad91c86ba779e1f2b0f8a27a Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 4 Dec 2024 20:15:55 +0200 Subject: [PATCH 026/204] Create the mapping explicitly, otherwise for 0 documents indices (#118015) the mapping will not contain the "value" field --- muted-tests.yml | 6 ------ .../xpack/esql/qa/rest/RequestIndexFilteringTestCase.java | 5 ++++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index e69588d6ec359..0b9027a406088 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -230,12 +230,6 @@ tests: - class: org.elasticsearch.xpack.esql.plugin.ClusterRequestTests method: testFallbackIndicesOptions issue: https://github.com/elastic/elasticsearch/issues/117937 -- class: org.elasticsearch.xpack.esql.qa.single_node.RequestIndexFilteringIT - method: testFieldExistsFilter_KeepWildcard - issue: https://github.com/elastic/elasticsearch/issues/117935 -- class: org.elasticsearch.xpack.esql.qa.multi_node.RequestIndexFilteringIT - method: testFieldExistsFilter_KeepWildcard - issue: https://github.com/elastic/elasticsearch/issues/117935 - class: org.elasticsearch.xpack.ml.integration.RegressionIT method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet issue: https://github.com/elastic/elasticsearch/issues/117805 diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java index 3314430d63eaa..406997b66dbf0 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java @@ -101,7 +101,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException { indexTimestampData(docsTest1, "test1", "2024-11-26", "id1"); indexTimestampData(docsTest2, "test2", "2023-11-26", "id2"); - // filter includes only test1. Columns are rows of test2 are filtered out + // filter includes only test1. Columns and rows of test2 are filtered out RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query("FROM test*"); Map result = runEsql(builder); assertMap( @@ -253,6 +253,9 @@ protected void indexTimestampData(int docs, String indexName, String date, Strin "@timestamp": { "type": "date" }, + "value": { + "type": "long" + }, "%differentiator_field_name%": { "type": "integer" } From b716a53084f9329d81f9629dd41fcce5999a6cd2 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 4 Dec 2024 10:25:51 -0800 Subject: [PATCH 027/204] Change deprecation.elasticsearch keyword to elasticsearch.deprecation (#117933) Changes the values of the "data_stream.dataset" and "event.dataset" fields to be "elasticsearch.deprecation" instead of "deprecation.elasticsearch". --- distribution/src/config/log4j2.properties | 2 +- docs/changelog/117933.yaml | 18 ++++++++++ .../common/logging/JsonLoggerTests.java | 34 +++++++++---------- .../logging/json_layout/log4j2.properties | 5 ++- .../common/logging/DeprecatedMessage.java | 2 +- .../xpack/deprecation/DeprecationHttpIT.java | 28 +++++++-------- .../logging/DeprecationIndexingComponent.java | 2 +- 7 files changed, 54 insertions(+), 37 deletions(-) create mode 100644 docs/changelog/117933.yaml diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties index 36b5b03d9a110..bde4d9d17fc17 100644 --- a/distribution/src/config/log4j2.properties +++ b/distribution/src/config/log4j2.properties @@ -63,7 +63,7 @@ appender.deprecation_rolling.name = deprecation_rolling appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json appender.deprecation_rolling.layout.type = ECSJsonLayout # Intentionally follows a different pattern to above -appender.deprecation_rolling.layout.dataset = deprecation.elasticsearch +appender.deprecation_rolling.layout.dataset = elasticsearch.deprecation appender.deprecation_rolling.filter.rate_limit.type = RateLimitingFilter appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz diff --git a/docs/changelog/117933.yaml b/docs/changelog/117933.yaml new file mode 100644 index 0000000000000..92ae31afa30dd --- /dev/null +++ b/docs/changelog/117933.yaml @@ -0,0 +1,18 @@ +pr: 117933 +summary: Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` +area: Infra/Logging +type: bug +issues: + - 83251 +breaking: + title: Deprecation logging value change for "data_stream.dataset" and "event.dataset" + area: Logging + details: |- + This change modifies the "data_stream.dataset" and "event.dataset" value for deprecation logging + to use the value `elasticsearch.deprecation` instead of `deprecation.elasticsearch`. This is now + consistent with other values where the name of the service is the first part of the key. + impact: |- + If you are directly consuming deprecation logs for "data_stream.dataset" and "event.dataset" and filtering on + this value, you will need to update your filters to use `elasticsearch.deprecation` instead of + `deprecation.elasticsearch`. + notable: false diff --git a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java index 1066bf1360e41..ed6205c7a5208 100644 --- a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java +++ b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java @@ -125,14 +125,14 @@ public void testDeprecatedMessageWithoutXOpaqueId() throws IOException { jsonLogs, contains( allOf( - hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), hasEntry("log.level", "CRITICAL"), hasEntry("log.logger", "org.elasticsearch.deprecation.test"), hasEntry("elasticsearch.cluster.name", "elasticsearch"), hasEntry("elasticsearch.node.name", "sample-name"), hasEntry("message", "deprecated message1"), hasEntry("data_stream.type", "logs"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasKey("ecs.version"), hasEntry(DeprecatedMessage.KEY_FIELD_NAME, "a key"), @@ -168,8 +168,8 @@ public void testCompatibleLog() throws Exception { contains( allOf( hasEntry("log.level", "CRITICAL"), - hasEntry("event.dataset", "deprecation.elasticsearch"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), hasEntry("log.logger", "org.elasticsearch.deprecation.test"), @@ -186,8 +186,8 @@ public void testCompatibleLog() throws Exception { allOf( hasEntry("log.level", "CRITICAL"), // event.dataset and data_stream.dataset have to be the same across the data stream - hasEntry("event.dataset", "deprecation.elasticsearch"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), hasEntry("log.logger", "org.elasticsearch.deprecation.test"), @@ -240,8 +240,8 @@ public void testParseFieldEmittingDeprecatedLogs() throws Exception { // deprecation log for field deprecated_name allOf( hasEntry("log.level", "WARN"), - hasEntry("event.dataset", "deprecation.elasticsearch"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), hasEntry("log.logger", "org.elasticsearch.deprecation.xcontent.ParseField"), @@ -258,8 +258,8 @@ public void testParseFieldEmittingDeprecatedLogs() throws Exception { // deprecation log for field deprecated_name2 (note it is not being throttled) allOf( hasEntry("log.level", "WARN"), - hasEntry("event.dataset", "deprecation.elasticsearch"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), hasEntry("log.logger", "org.elasticsearch.deprecation.xcontent.ParseField"), @@ -276,8 +276,8 @@ public void testParseFieldEmittingDeprecatedLogs() throws Exception { // compatible log line allOf( hasEntry("log.level", "CRITICAL"), - hasEntry("event.dataset", "deprecation.elasticsearch"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), hasEntry("log.logger", "org.elasticsearch.deprecation.xcontent.ParseField"), @@ -327,14 +327,14 @@ public void testDeprecatedMessage() throws Exception { jsonLogs, contains( allOf( - hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), hasEntry("log.level", "WARN"), hasEntry("log.logger", "org.elasticsearch.deprecation.test"), hasEntry("elasticsearch.cluster.name", "elasticsearch"), hasEntry("elasticsearch.node.name", "sample-name"), hasEntry("message", "deprecated message1"), hasEntry("data_stream.type", "logs"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasKey("ecs.version"), hasEntry(DeprecatedMessage.KEY_FIELD_NAME, "someKey"), @@ -579,7 +579,7 @@ public void testDuplicateLogMessages() throws Exception { jsonLogs, contains( allOf( - hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), hasEntry("log.level", "CRITICAL"), hasEntry("log.logger", "org.elasticsearch.deprecation.test"), hasEntry("elasticsearch.cluster.name", "elasticsearch"), @@ -612,7 +612,7 @@ public void testDuplicateLogMessages() throws Exception { jsonLogs, contains( allOf( - hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), hasEntry("log.level", "CRITICAL"), hasEntry("log.logger", "org.elasticsearch.deprecation.test"), hasEntry("elasticsearch.cluster.name", "elasticsearch"), @@ -622,7 +622,7 @@ public void testDuplicateLogMessages() throws Exception { hasEntry("elasticsearch.event.category", "other") ), allOf( - hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), hasEntry("log.level", "CRITICAL"), hasEntry("log.logger", "org.elasticsearch.deprecation.test"), hasEntry("elasticsearch.cluster.name", "elasticsearch"), diff --git a/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties b/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties index 46baac4f1433c..b00caca66d03c 100644 --- a/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties +++ b/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties @@ -15,14 +15,13 @@ appender.deprecated.name = deprecated appender.deprecated.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecated.json # Intentionally follows a different pattern to above appender.deprecated.layout.type = ECSJsonLayout -appender.deprecated.layout.dataset = deprecation.elasticsearch +appender.deprecated.layout.dataset = elasticsearch.deprecation appender.deprecated.filter.rate_limit.type = RateLimitingFilter appender.deprecatedconsole.type = Console appender.deprecatedconsole.name = deprecatedconsole appender.deprecatedconsole.layout.type = ECSJsonLayout -# Intentionally follows a different pattern to above -appender.deprecatedconsole.layout.dataset = deprecation.elasticsearch +appender.deprecatedconsole.layout.dataset = elasticsearch.deprecation appender.deprecatedconsole.filter.rate_limit.type = RateLimitingFilter diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java index 0bcde14fcf19a..ca89313e59de2 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java @@ -57,7 +57,7 @@ private static ESLogMessage getEsLogMessage( String messagePattern, Object[] args ) { - ESLogMessage esLogMessage = new ESLogMessage(messagePattern, args).field("data_stream.dataset", "deprecation.elasticsearch") + ESLogMessage esLogMessage = new ESLogMessage(messagePattern, args).field("data_stream.dataset", "elasticsearch.deprecation") .field("data_stream.type", "logs") .field("data_stream.namespace", "default") .field(KEY_FIELD_NAME, key) diff --git a/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java b/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java index 4a17c2abbd797..2136129a671c8 100644 --- a/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java +++ b/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java @@ -339,12 +339,12 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception { hasEntry("elasticsearch.event.category", "settings"), hasKey("elasticsearch.node.id"), hasKey("elasticsearch.node.name"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), hasKey("ecs.version"), hasEntry(KEY_FIELD_NAME, "deprecated_settings"), - hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), hasEntry("log.level", "WARN"), hasKey("log.logger"), hasEntry("message", "[deprecated_settings] usage is deprecated. use [settings] instead") @@ -357,12 +357,12 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception { hasEntry("elasticsearch.event.category", "api"), hasKey("elasticsearch.node.id"), hasKey("elasticsearch.node.name"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), hasKey("ecs.version"), hasEntry(KEY_FIELD_NAME, "deprecated_route_GET_/_test_cluster/deprecated_settings"), - hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), hasEntry("log.level", "WARN"), hasKey("log.logger"), hasEntry("message", "[/_test_cluster/deprecated_settings] exists for deprecated tests") @@ -402,12 +402,12 @@ public void testDeprecationCriticalWarnMessagesCanBeIndexed() throws Exception { hasEntry("elasticsearch.event.category", "settings"), hasKey("elasticsearch.node.id"), hasKey("elasticsearch.node.name"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), hasKey("ecs.version"), hasEntry(KEY_FIELD_NAME, "deprecated_critical_settings"), - hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), hasEntry("log.level", "CRITICAL"), hasKey("log.logger"), hasEntry("message", "[deprecated_settings] usage is deprecated. use [settings] instead") @@ -443,12 +443,12 @@ public void testDeprecationWarnMessagesCanBeIndexed() throws Exception { hasEntry("elasticsearch.event.category", "settings"), hasKey("elasticsearch.node.id"), hasKey("elasticsearch.node.name"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), hasKey("ecs.version"), hasEntry(KEY_FIELD_NAME, "deprecated_warn_settings"), - hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), hasEntry("log.level", "WARN"), hasKey("log.logger"), hasEntry("message", "[deprecated_warn_settings] usage is deprecated but won't be breaking in next version") @@ -461,12 +461,12 @@ public void testDeprecationWarnMessagesCanBeIndexed() throws Exception { hasEntry("elasticsearch.event.category", "api"), hasKey("elasticsearch.node.id"), hasKey("elasticsearch.node.name"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), hasKey("ecs.version"), hasEntry(KEY_FIELD_NAME, "deprecated_route_GET_/_test_cluster/deprecated_settings"), - hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), hasEntry("log.level", "WARN"), hasKey("log.logger"), hasEntry("message", "[/_test_cluster/deprecated_settings] exists for deprecated tests") @@ -619,12 +619,12 @@ public void testCompatibleMessagesCanBeIndexed() throws Exception { hasEntry("elasticsearch.event.category", "compatible_api"), hasKey("elasticsearch.node.id"), hasKey("elasticsearch.node.name"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), hasKey("ecs.version"), hasEntry(KEY_FIELD_NAME, "compatible_key"), - hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), hasEntry("log.level", "CRITICAL"), hasKey("log.logger"), hasEntry("message", "You are using a compatible API for this request") @@ -637,12 +637,12 @@ public void testCompatibleMessagesCanBeIndexed() throws Exception { hasEntry("elasticsearch.event.category", "compatible_api"), hasKey("elasticsearch.node.id"), hasKey("elasticsearch.node.name"), - hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "elasticsearch.deprecation"), hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), hasKey("ecs.version"), hasEntry(KEY_FIELD_NAME, "deprecated_route_GET_/_test_cluster/compat_only"), - hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("event.dataset", "elasticsearch.deprecation"), hasEntry("log.level", "CRITICAL"), hasKey("log.logger"), hasEntry("message", "[/_test_cluster/deprecated_settings] exists for deprecated tests") diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java index 29041b0c58434..507f4b18c79e9 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java @@ -91,7 +91,7 @@ private DeprecationIndexingComponent( final Configuration configuration = context.getConfiguration(); final EcsLayout ecsLayout = ECSJsonLayout.newBuilder() - .setDataset("deprecation.elasticsearch") + .setDataset("elasticsearch.deprecation") .setConfiguration(configuration) .build(); From 9a81eb2dbe17c3d8ce2920aec33a1d7a89a93c38 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 4 Dec 2024 14:10:32 -0500 Subject: [PATCH 028/204] Indicate that rescore isn't allowed with retrievers, yet (#118019) --- docs/reference/search/retriever.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc index b90b7e312c790..cb04d4fb6fbf1 100644 --- a/docs/reference/search/retriever.asciidoc +++ b/docs/reference/search/retriever.asciidoc @@ -765,11 +765,11 @@ clauses in a <>. [[retriever-restrictions]] ==== Restrictions on search parameters when specifying a retriever -When a retriever is specified as part of a search, the following elements are not allowed at the top-level. -Instead they are only allowed as elements of specific retrievers: +When a retriever is specified as part of a search, the following elements are not allowed at the top-level: * <> * <> * <> * <> * <> +* <> From eb0020f0558faefbadab1ee952e3a15410f3eb34 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 4 Dec 2024 21:04:11 +0100 Subject: [PATCH 029/204] Introduce more parallelism into cross cluster test bootstrapping (#117820) We can parallelize starting the clusters and a few other things to effectively speed up these tests by 2x which comes out to about a minute of execution time saved for all of those in :server:internalClusterTests on my workstation. --- .../ResolveClusterDataStreamIT.java | 2 +- .../action/CrossClusterPainlessExecuteIT.java | 2 +- .../index/reindex/CrossClusterReindexIT.java | 2 +- .../admin/cluster/remote/RemoteInfoIT.java | 3 +- .../cluster/stats/ClusterStatsRemoteIT.java | 3 +- .../action/search/CCSPointInTimeIT.java | 2 +- .../indices/cluster/ResolveClusterIT.java | 3 +- .../search/ccs/CCSCanMatchIT.java | 2 +- .../search/ccs/CCSUsageTelemetryIT.java | 72 ++++++++++--------- .../search/ccs/CrossClusterIT.java | 2 +- .../search/ccs/CrossClusterSearchIT.java | 2 +- .../search/ccs/CrossClusterSearchLeakIT.java | 2 +- .../fieldcaps/CCSFieldCapabilitiesIT.java | 2 +- .../retriever/MinimalCompoundRetrieverIT.java | 3 +- .../test/AbstractMultiClustersTestCase.java | 29 ++++---- .../CCSUsageTelemetryAsyncSearchIT.java | 2 +- .../search/CrossClusterAsyncSearchIT.java | 2 +- .../xpack/core/termsenum/CCSTermsEnumIT.java | 2 +- .../esql/action/CrossClusterAsyncQueryIT.java | 2 +- ...ossClusterEnrichUnavailableClustersIT.java | 2 +- ...CrossClusterQueryUnavailableRemotesIT.java | 2 +- .../action/CrossClustersCancellationIT.java | 2 +- .../esql/action/CrossClustersEnrichIT.java | 2 +- .../esql/action/CrossClustersQueryIT.java | 2 +- .../xpack/ml/integration/DatafeedCcsIT.java | 2 +- .../checkpoint/TransformCCSCanMatchIT.java | 2 +- 26 files changed, 80 insertions(+), 73 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java index 4c85958498da0..aa6ecf35e06fa 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java @@ -78,7 +78,7 @@ public class ResolveClusterDataStreamIT extends AbstractMultiClustersTestCase { private static long LATEST_TIMESTAMP = 1691348820000L; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2); } diff --git a/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/action/CrossClusterPainlessExecuteIT.java b/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/action/CrossClusterPainlessExecuteIT.java index 4669ab25f5d8c..b21cabad9290c 100644 --- a/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/action/CrossClusterPainlessExecuteIT.java +++ b/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/action/CrossClusterPainlessExecuteIT.java @@ -54,7 +54,7 @@ public class CrossClusterPainlessExecuteIT extends AbstractMultiClustersTestCase private static final String KEYWORD_FIELD = "my_field"; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER); } diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java index 8b94337141243..4624393e9fb60 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java @@ -36,7 +36,7 @@ protected boolean reuseClusters() { } @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoIT.java index 25678939cb375..9e578faaac70c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoIT.java @@ -15,7 +15,6 @@ import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.NodeRoles; -import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -24,7 +23,7 @@ public class RemoteInfoIT extends AbstractMultiClustersTestCase { @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { if (randomBoolean()) { return List.of(); } else { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRemoteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRemoteIT.java index 6cc9824245247..5f4315abff405 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRemoteIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRemoteIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.test.InternalTestCluster; import org.junit.Assert; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -51,7 +50,7 @@ protected boolean reuseClusters() { } @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE1, REMOTE2); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/CCSPointInTimeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/CCSPointInTimeIT.java index ed92e7704f4ba..7a75313d44189 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/CCSPointInTimeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/CCSPointInTimeIT.java @@ -44,7 +44,7 @@ public class CCSPointInTimeIT extends AbstractMultiClustersTestCase { public static final String REMOTE_CLUSTER = "remote_cluster"; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java index 1a6674edc5147..4bdc5d63f4a2f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java @@ -28,7 +28,6 @@ import org.elasticsearch.transport.RemoteClusterAware; import java.io.IOException; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -54,7 +53,7 @@ public class ResolveClusterIT extends AbstractMultiClustersTestCase { private static long LATEST_TIMESTAMP = 1691348820000L; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSCanMatchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSCanMatchIT.java index 3f354baace85a..ce898d9be15ca 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSCanMatchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSCanMatchIT.java @@ -55,7 +55,7 @@ public class CCSCanMatchIT extends AbstractMultiClustersTestCase { static final String REMOTE_CLUSTER = "cluster_a"; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of("cluster_a"); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java index c9d34dbf14015..9c1daccd2cc9e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java @@ -11,16 +11,19 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot; import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.Result; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.search.ClosePointInTimeRequest; import org.elasticsearch.action.search.OpenPointInTimeRequest; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.TransportClosePointInTimeAction; import org.elasticsearch.action.search.TransportOpenPointInTimeAction; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.RefCountingListener; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -78,7 +81,7 @@ protected boolean reuseClusters() { } @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE1, REMOTE2); } @@ -126,12 +129,9 @@ private CCSTelemetrySnapshot getTelemetryFromFailedSearch(SearchRequest searchRe // We want to send search to a specific node (we don't care which one) so that we could // collect the CCS telemetry from it later String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); - PlainActionFuture queryFuture = new PlainActionFuture<>(); - cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest, queryFuture); - assertBusy(() -> assertTrue(queryFuture.isDone())); // We expect failure, but we don't care too much which failure it is in this test - ExecutionException ee = expectThrows(ExecutionException.class, queryFuture::get); + ExecutionException ee = expectThrows(ExecutionException.class, cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest)::get); assertNotNull(ee.getCause()); return getTelemetrySnapshot(nodeName); @@ -637,56 +637,62 @@ private CCSTelemetrySnapshot getTelemetrySnapshot(String nodeName) { return usage.getCcsUsageHolder().getCCSTelemetrySnapshot(); } - private Map setupClusters() { + private Map setupClusters() throws ExecutionException, InterruptedException { String localIndex = "demo"; + String remoteIndex = "prod"; int numShardsLocal = randomIntBetween(2, 10); Settings localSettings = indexSettings(numShardsLocal, randomIntBetween(0, 1)).build(); - assertAcked( + final PlainActionFuture future = new PlainActionFuture<>(); + try (RefCountingListener refCountingListener = new RefCountingListener(future)) { client(LOCAL_CLUSTER).admin() .indices() .prepareCreate(localIndex) .setSettings(localSettings) .setMapping("@timestamp", "type=date", "f", "type=text") - ); - indexDocs(client(LOCAL_CLUSTER), localIndex); - - String remoteIndex = "prod"; - int numShardsRemote = randomIntBetween(2, 10); - for (String clusterAlias : remoteClusterAlias()) { - final InternalTestCluster remoteCluster = cluster(clusterAlias); - remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(2, 3)); - assertAcked( + .execute(refCountingListener.acquire(r -> { + assertAcked(r); + indexDocs(client(LOCAL_CLUSTER), localIndex, refCountingListener.acquire()); + })); + + int numShardsRemote = randomIntBetween(2, 10); + var remotes = remoteClusterAlias(); + runInParallel(remotes.size(), i -> { + final String clusterAlias = remotes.get(i); + final InternalTestCluster remoteCluster = cluster(clusterAlias); + remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(2, 3)); client(clusterAlias).admin() .indices() .prepareCreate(remoteIndex) .setSettings(indexSettings(numShardsRemote, randomIntBetween(0, 1))) .setMapping("@timestamp", "type=date", "f", "type=text") - ); - assertFalse( - client(clusterAlias).admin() - .cluster() - .prepareHealth(TEST_REQUEST_TIMEOUT, remoteIndex) - .setWaitForYellowStatus() - .setTimeout(TimeValue.timeValueSeconds(10)) - .get() - .isTimedOut() - ); - indexDocs(client(clusterAlias), remoteIndex); + .execute(refCountingListener.acquire(r -> { + assertAcked(r); + client(clusterAlias).admin() + .cluster() + .prepareHealth(TEST_REQUEST_TIMEOUT, remoteIndex) + .setWaitForYellowStatus() + .setTimeout(TimeValue.timeValueSeconds(10)) + .execute(refCountingListener.acquire(healthResponse -> { + assertFalse(healthResponse.isTimedOut()); + indexDocs(client(clusterAlias), remoteIndex, refCountingListener.acquire()); + })); + })); + }); } - + future.get(); Map clusterInfo = new HashMap<>(); clusterInfo.put("local.index", localIndex); clusterInfo.put("remote.index", remoteIndex); return clusterInfo; } - private int indexDocs(Client client, String index) { + private void indexDocs(Client client, String index, ActionListener listener) { int numDocs = between(5, 20); + final BulkRequestBuilder bulkRequest = client.prepareBulk(); for (int i = 0; i < numDocs; i++) { - client.prepareIndex(index).setSource("f", "v", "@timestamp", randomNonNegativeLong()).get(); + bulkRequest.add(client.prepareIndex(index).setSource("f", "v", "@timestamp", randomNonNegativeLong())); } - client.admin().indices().prepareRefresh(index).get(); - return numDocs; + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).execute(listener.safeMap(r -> null)); } /** diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java index cb4d0681cdb23..57a9f8131ac2d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java @@ -86,7 +86,7 @@ public class CrossClusterIT extends AbstractMultiClustersTestCase { @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of("cluster_a"); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java index 63eece88a53fc..823d3198bc7a2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java @@ -60,7 +60,7 @@ public class CrossClusterSearchIT extends AbstractMultiClustersTestCase { private static long LATEST_TIMESTAMP = 1691348820000L; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java index 8b493782d55b5..e8a3df353a01e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java @@ -38,7 +38,7 @@ public class CrossClusterSearchLeakIT extends AbstractMultiClustersTestCase { @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of("cluster_a"); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java index 56b34f9b1dfec..f29cff98c6495 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java @@ -34,7 +34,7 @@ public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase { @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of("remote_cluster"); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java index 97aa428822fae..8dc37bad675e8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java @@ -26,7 +26,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -43,7 +42,7 @@ public class MinimalCompoundRetrieverIT extends AbstractMultiClustersTestCase { private static final String REMOTE_CLUSTER = "cluster_a"; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java index ea82c9d21ab89..b4f91f68b8bb7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java @@ -36,10 +36,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.stream.Collectors; @@ -58,7 +58,7 @@ public abstract class AbstractMultiClustersTestCase extends ESTestCase { private static volatile ClusterGroup clusterGroup; - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return randomSubsetOf(List.of("cluster-a", "cluster-b")); } @@ -100,17 +100,18 @@ public final void startClusters() throws Exception { return; } stopClusters(); - final Map clusters = new HashMap<>(); + final Map clusters = new ConcurrentHashMap<>(); final List clusterAliases = new ArrayList<>(remoteClusterAlias()); clusterAliases.add(LOCAL_CLUSTER); - for (String clusterAlias : clusterAliases) { + final List> mockPlugins = List.of( + MockHttpTransport.TestPlugin.class, + MockTransportService.TestPlugin.class, + getTestTransportPlugin() + ); + runInParallel(clusterAliases.size(), i -> { + String clusterAlias = clusterAliases.get(i); final String clusterName = clusterAlias.equals(LOCAL_CLUSTER) ? "main-cluster" : clusterAlias; final int numberOfNodes = randomIntBetween(1, 3); - final List> mockPlugins = List.of( - MockHttpTransport.TestPlugin.class, - MockTransportService.TestPlugin.class, - getTestTransportPlugin() - ); final Collection> nodePlugins = nodePlugins(clusterAlias); final NodeConfigurationSource nodeConfigurationSource = nodeConfigurationSource(nodeSettings(), nodePlugins); @@ -128,10 +129,14 @@ public final void startClusters() throws Exception { mockPlugins, Function.identity() ); - cluster.beforeTest(random()); + try { + cluster.beforeTest(random()); + } catch (Exception e) { + throw new RuntimeException(e); + } clusters.put(clusterAlias, cluster); - } - clusterGroup = new ClusterGroup(clusters); + }); + clusterGroup = new ClusterGroup(Map.copyOf(clusters)); configureAndConnectsToRemoteClusters(); } diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CCSUsageTelemetryAsyncSearchIT.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CCSUsageTelemetryAsyncSearchIT.java index 65f9f13846126..1b19f6f04693b 100644 --- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CCSUsageTelemetryAsyncSearchIT.java +++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CCSUsageTelemetryAsyncSearchIT.java @@ -60,7 +60,7 @@ protected boolean reuseClusters() { } @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE1, REMOTE2); } diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java index 3b5647da1399f..2a8daf8bfe12c 100644 --- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java +++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java @@ -88,7 +88,7 @@ public class CrossClusterAsyncSearchIT extends AbstractMultiClustersTestCase { private static final long LATEST_TIMESTAMP = 1691348820000L; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER); } diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/termsenum/CCSTermsEnumIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/termsenum/CCSTermsEnumIT.java index 157628be9fbc9..f5c070073d9b5 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/termsenum/CCSTermsEnumIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/termsenum/CCSTermsEnumIT.java @@ -26,7 +26,7 @@ public class CCSTermsEnumIT extends AbstractMultiClustersTestCase { @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of("remote_cluster"); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java index 440582dcfbb45..c8206621de419 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java @@ -66,7 +66,7 @@ public class CrossClusterAsyncQueryIT extends AbstractMultiClustersTestCase { private static final String INDEX_WITH_RUNTIME_MAPPING = "blocking"; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java index d142752d0c408..5c3e1974e924f 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java @@ -53,7 +53,7 @@ public class CrossClusterEnrichUnavailableClustersIT extends AbstractMultiCluste public static String REMOTE_CLUSTER_2 = "c2"; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java index 0f1aa8541fdd9..d1c9b5cfb2ac7 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java @@ -42,7 +42,7 @@ public class CrossClusterQueryUnavailableRemotesIT extends AbstractMultiClusters private static final String REMOTE_CLUSTER_2 = "cluster-b"; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java index f29f79976dc0d..5291ad3b0d039 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java @@ -55,7 +55,7 @@ public class CrossClustersCancellationIT extends AbstractMultiClustersTestCase { private static final String REMOTE_CLUSTER = "cluster-a"; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java index e8e9f45694e9c..57f85751999a5 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java @@ -64,7 +64,7 @@ public class CrossClustersEnrichIT extends AbstractMultiClustersTestCase { @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of("c1", "c2"); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java index 596c70e57ccd6..46bbad5551e6b 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java @@ -67,7 +67,7 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { private static String REMOTE_INDEX = "logs-2"; @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2); } diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedCcsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedCcsIT.java index 139d1b074c7b2..e437c91c8e50e 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedCcsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/DatafeedCcsIT.java @@ -94,7 +94,7 @@ protected Collection> nodePlugins(String clusterAlias) { } @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER); } diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java index 208da4177fd4c..e4e577299d0d7 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java @@ -385,7 +385,7 @@ protected NamedXContentRegistry xContentRegistry() { } @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER); } From a2ea8549c8a0fe7efd22617a8fd6bf74a2b55cd4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 5 Dec 2024 08:20:22 +1100 Subject: [PATCH 030/204] Mute org.elasticsearch.upgrades.QueryBuilderBWCIT testQueryBuilderBWC {cluster=UPGRADED} #116990 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 0b9027a406088..1f2f5735c4a7c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -239,6 +239,9 @@ tests: - class: org.elasticsearch.packaging.test.CertGenCliTests method: test40RunWithCert issue: https://github.com/elastic/elasticsearch/issues/117955 +- class: org.elasticsearch.upgrades.QueryBuilderBWCIT + method: testQueryBuilderBWC {cluster=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/116990 # Examples: # From 9df4a4968f297d1407743a2f55be2f1feec52f70 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 5 Dec 2024 08:20:32 +1100 Subject: [PATCH 031/204] Mute org.elasticsearch.xpack.restart.QueryBuilderBWCIT testQueryBuilderBWC {p0=UPGRADED} #116989 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 1f2f5735c4a7c..0b400f420e86b 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -242,6 +242,9 @@ tests: - class: org.elasticsearch.upgrades.QueryBuilderBWCIT method: testQueryBuilderBWC {cluster=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/116990 +- class: org.elasticsearch.xpack.restart.QueryBuilderBWCIT + method: testQueryBuilderBWC {p0=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/116989 # Examples: # From 4e8e9731567d66ed4cb7c3a2c7ed5af744e32ae7 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 4 Dec 2024 22:22:36 +0100 Subject: [PATCH 032/204] Remove some dead code from SearchPhase and friends (#116645) The separate `onFailure` is unnecessary, just fail the phase like we do elsewhere. Also make utility method static. --- .../action/search/AbstractSearchAsyncAction.java | 9 +-------- .../elasticsearch/action/search/ExpandSearchPhase.java | 8 ++++++-- .../action/search/MockSearchPhaseContext.java | 5 ----- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index 09fb70fb06ba4..800193e258dba 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -739,7 +739,7 @@ void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connecti * @see #onShardFailure(int, SearchShardTarget, Exception) * @see #onShardResult(SearchPhaseResult, SearchShardIterator) */ - final void onPhaseDone() { // as a tribute to @kimchy aka. finishHim() + private void onPhaseDone() { // as a tribute to @kimchy aka. finishHim() executeNextPhase(this, this::getNextPhase); } @@ -762,13 +762,6 @@ public final void execute(Runnable command) { executor.execute(command); } - /** - * Notifies the top-level listener of the provided exception - */ - public void onFailure(Exception e) { - listener.onFailure(e); - } - /** * Builds an request for the initial search phase. * diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 8feed2aea00b0..e8d94c32bdcc7 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -102,7 +102,7 @@ private void doRun() { for (InnerHitBuilder innerHitBuilder : innerHitBuilders) { MultiSearchResponse.Item item = it.next(); if (item.isFailure()) { - context.onPhaseFailure(this, "failed to expand hits", item.getFailure()); + phaseFailure(item.getFailure()); return; } SearchHits innerHits = item.getResponse().getHits(); @@ -119,7 +119,11 @@ private void doRun() { } } onPhaseDone(); - }, context::onFailure)); + }, this::phaseFailure)); + } + + private void phaseFailure(Exception ex) { + context.onPhaseFailure(this, "failed to expand hits", ex); } private static SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder options, CollapseBuilder innerCollapseBuilder) { diff --git a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java index 484b3c6b386fd..7a38858d8477a 100644 --- a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java +++ b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java @@ -154,11 +154,6 @@ protected void executePhaseOnShard( }, shardIt); } - @Override - public void onFailure(Exception e) { - Assert.fail("should not be called"); - } - @Override public void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) { releasedSearchContexts.add(contextId); From 2a677db7d3ba3f4bdc9f48f3e3c287c32604d189 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:48:39 +1100 Subject: [PATCH 033/204] Mute org.elasticsearch.index.reindex.ReindexNodeShutdownIT testReindexWithShutdown #118040 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 0b400f420e86b..f525607a848d0 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -245,6 +245,9 @@ tests: - class: org.elasticsearch.xpack.restart.QueryBuilderBWCIT method: testQueryBuilderBWC {p0=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/116989 +- class: org.elasticsearch.index.reindex.ReindexNodeShutdownIT + method: testReindexWithShutdown + issue: https://github.com/elastic/elasticsearch/issues/118040 # Examples: # From fa48715f85eb6dae0eee93153b34ead02278b553 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 5 Dec 2024 17:30:01 +1100 Subject: [PATCH 034/204] Mute org.elasticsearch.xpack.test.rest.XPackRestIT test {p0=esql/60_usage/Basic ESQL usage output (telemetry) non-snapshot version} #117862 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index f525607a848d0..de18c08337e11 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -248,6 +248,9 @@ tests: - class: org.elasticsearch.index.reindex.ReindexNodeShutdownIT method: testReindexWithShutdown issue: https://github.com/elastic/elasticsearch/issues/118040 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=esql/60_usage/Basic ESQL usage output (telemetry) non-snapshot version} + issue: https://github.com/elastic/elasticsearch/issues/117862 # Examples: # From 0d4c0f208087ca84d9727b777a366bb1448f2a4c Mon Sep 17 00:00:00 2001 From: Panagiotis Bailis Date: Thu, 5 Dec 2024 09:06:53 +0200 Subject: [PATCH 035/204] Fix for propagating filters from compound to inner retrievers (#117914) --- docs/changelog/117914.yaml | 5 ++ .../retriever/CompoundRetrieverBuilder.java | 32 +++++--- .../search/retriever/KnnRetrieverBuilder.java | 3 +- .../retriever/RankDocsRetrieverBuilder.java | 16 +--- .../RankDocsRetrieverBuilderTests.java | 7 +- .../vectors/TestQueryVectorBuilderPlugin.java | 8 +- .../TestCompoundRetrieverBuilder.java | 10 ++- .../retriever/QueryRuleRetrieverBuilder.java | 15 +++- .../TextSimilarityRankRetrieverBuilder.java | 7 +- .../xpack/rank/rrf/RRFRetrieverBuilderIT.java | 38 +++++++++- .../xpack/rank/rrf/RRFFeatures.java | 6 ++ .../xpack/rank/rrf/RRFRetrieverBuilder.java | 7 +- ...rrf_retriever_search_api_compatibility.yml | 74 +++++++++++++++++++ 13 files changed, 180 insertions(+), 48 deletions(-) create mode 100644 docs/changelog/117914.yaml diff --git a/docs/changelog/117914.yaml b/docs/changelog/117914.yaml new file mode 100644 index 0000000000000..da58ed7bb04b7 --- /dev/null +++ b/docs/changelog/117914.yaml @@ -0,0 +1,5 @@ +pr: 117914 +summary: Fix for propagating filters from compound to inner retrievers +area: Ranking +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java index db839de9f573a..2ab6395db73b5 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.TransportMultiSearchAction; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.rest.RestStatus; @@ -46,6 +47,8 @@ */ public abstract class CompoundRetrieverBuilder> extends RetrieverBuilder { + public static final NodeFeature INNER_RETRIEVERS_FILTER_SUPPORT = new NodeFeature("inner_retrievers_filter_support"); + public record RetrieverSource(RetrieverBuilder retriever, SearchSourceBuilder source) {} protected final int rankWindowSize; @@ -64,9 +67,9 @@ public T addChild(RetrieverBuilder retrieverBuilder) { /** * Returns a clone of the original retriever, replacing the sub-retrievers with - * the provided {@code newChildRetrievers}. + * the provided {@code newChildRetrievers} and the filters with the {@code newPreFilterQueryBuilders}. */ - protected abstract T clone(List newChildRetrievers); + protected abstract T clone(List newChildRetrievers, List newPreFilterQueryBuilders); /** * Combines the provided {@code rankResults} to return the final top documents. @@ -85,13 +88,25 @@ public final RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOExceptio } // Rewrite prefilters - boolean hasChanged = false; + // We eagerly rewrite prefilters, because some of the innerRetrievers + // could be compound too, so we want to propagate all the necessary filter information to them + // and have it available as part of their own rewrite step var newPreFilters = rewritePreFilters(ctx); - hasChanged |= newPreFilters != preFilterQueryBuilders; + if (newPreFilters != preFilterQueryBuilders) { + return clone(innerRetrievers, newPreFilters); + } + boolean hasChanged = false; // Rewrite retriever sources List newRetrievers = new ArrayList<>(); for (var entry : innerRetrievers) { + // we propagate the filters only for compound retrievers as they won't be attached through + // the createSearchSourceBuilder. + // We could remove this check, but we would end up adding the same filters + // multiple times in case an inner retriever rewrites itself, when we re-enter to rewrite + if (entry.retriever.isCompound() && false == preFilterQueryBuilders.isEmpty()) { + entry.retriever.getPreFilterQueryBuilders().addAll(preFilterQueryBuilders); + } RetrieverBuilder newRetriever = entry.retriever.rewrite(ctx); if (newRetriever != entry.retriever) { newRetrievers.add(new RetrieverSource(newRetriever, null)); @@ -106,7 +121,7 @@ public final RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOExceptio } } if (hasChanged) { - return clone(newRetrievers); + return clone(newRetrievers, newPreFilters); } // execute searches @@ -166,12 +181,7 @@ public void onFailure(Exception e) { }); }); - return new RankDocsRetrieverBuilder( - rankWindowSize, - newRetrievers.stream().map(s -> s.retriever).toList(), - results::get, - newPreFilters - ); + return new RankDocsRetrieverBuilder(rankWindowSize, newRetrievers.stream().map(s -> s.retriever).toList(), results::get); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index 8be9a78dae154..f1464c41ca3be 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -184,8 +184,7 @@ public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { ll.onResponse(null); })); }); - var rewritten = new KnnRetrieverBuilder(this, () -> toSet.get(), null); - return rewritten; + return new KnnRetrieverBuilder(this, () -> toSet.get(), null); } return super.rewrite(ctx); } diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java index 02f890f51d011..4d3f3fefd4462 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java @@ -33,19 +33,13 @@ public class RankDocsRetrieverBuilder extends RetrieverBuilder { final List sources; final Supplier rankDocs; - public RankDocsRetrieverBuilder( - int rankWindowSize, - List sources, - Supplier rankDocs, - List preFilterQueryBuilders - ) { + public RankDocsRetrieverBuilder(int rankWindowSize, List sources, Supplier rankDocs) { this.rankWindowSize = rankWindowSize; this.rankDocs = rankDocs; if (sources == null || sources.isEmpty()) { throw new IllegalArgumentException("sources must not be null or empty"); } this.sources = sources; - this.preFilterQueryBuilders = preFilterQueryBuilders; } @Override @@ -73,10 +67,6 @@ private boolean sourceShouldRewrite(QueryRewriteContext ctx) throws IOException @Override public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { assert false == sourceShouldRewrite(ctx) : "retriever sources should be rewritten first"; - var rewrittenFilters = rewritePreFilters(ctx); - if (rewrittenFilters != preFilterQueryBuilders) { - return new RankDocsRetrieverBuilder(rankWindowSize, sources, rankDocs, rewrittenFilters); - } return this; } @@ -94,7 +84,7 @@ public QueryBuilder topDocsQuery() { boolQuery.should(query); } } - // ignore prefilters of this level, they are already propagated to children + // ignore prefilters of this level, they were already propagated to children return boolQuery; } @@ -133,7 +123,7 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder } else { rankQuery = new RankDocsQueryBuilder(rankDocResults, null, false); } - // ignore prefilters of this level, they are already propagated to children + // ignore prefilters of this level, they were already propagated to children searchSourceBuilder.query(rankQuery); if (sourceHasMinScore()) { searchSourceBuilder.minScore(this.minScore() == null ? Float.MIN_VALUE : this.minScore()); diff --git a/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java b/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java index af6782c45dce8..ccf33c0b71b6b 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java @@ -95,12 +95,7 @@ private List preFilters(QueryRewriteContext queryRewriteContext) t } private RankDocsRetrieverBuilder createRandomRankDocsRetrieverBuilder(QueryRewriteContext queryRewriteContext) throws IOException { - return new RankDocsRetrieverBuilder( - randomIntBetween(1, 100), - innerRetrievers(queryRewriteContext), - rankDocsSupplier(), - preFilters(queryRewriteContext) - ); + return new RankDocsRetrieverBuilder(randomIntBetween(1, 100), innerRetrievers(queryRewriteContext), rankDocsSupplier()); } public void testExtractToSearchSourceBuilder() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/vectors/TestQueryVectorBuilderPlugin.java b/server/src/test/java/org/elasticsearch/search/vectors/TestQueryVectorBuilderPlugin.java index c47c8c16f6a2f..5733a51bb7e9c 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/TestQueryVectorBuilderPlugin.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/TestQueryVectorBuilderPlugin.java @@ -27,9 +27,9 @@ /** * A SearchPlugin to exercise query vector builder */ -class TestQueryVectorBuilderPlugin implements SearchPlugin { +public class TestQueryVectorBuilderPlugin implements SearchPlugin { - static class TestQueryVectorBuilder implements QueryVectorBuilder { + public static class TestQueryVectorBuilder implements QueryVectorBuilder { private static final String NAME = "test_query_vector_builder"; private static final ParseField QUERY_VECTOR = new ParseField("query_vector"); @@ -47,11 +47,11 @@ static class TestQueryVectorBuilder implements QueryVectorBuilder { private List vectorToBuild; - TestQueryVectorBuilder(List vectorToBuild) { + public TestQueryVectorBuilder(List vectorToBuild) { this.vectorToBuild = vectorToBuild; } - TestQueryVectorBuilder(float[] expected) { + public TestQueryVectorBuilder(float[] expected) { this.vectorToBuild = new ArrayList<>(expected.length); for (float f : expected) { vectorToBuild.add(f); diff --git a/test/framework/src/main/java/org/elasticsearch/search/retriever/TestCompoundRetrieverBuilder.java b/test/framework/src/main/java/org/elasticsearch/search/retriever/TestCompoundRetrieverBuilder.java index 9f199aa7f3ef8..4a5f280c10a99 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/retriever/TestCompoundRetrieverBuilder.java +++ b/test/framework/src/main/java/org/elasticsearch/search/retriever/TestCompoundRetrieverBuilder.java @@ -10,6 +10,7 @@ package org.elasticsearch.search.retriever; import org.apache.lucene.search.ScoreDoc; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.xcontent.XContentBuilder; @@ -23,16 +24,17 @@ public class TestCompoundRetrieverBuilder extends CompoundRetrieverBuilder(), rankWindowSize); + this(new ArrayList<>(), rankWindowSize, new ArrayList<>()); } - TestCompoundRetrieverBuilder(List childRetrievers, int rankWindowSize) { + TestCompoundRetrieverBuilder(List childRetrievers, int rankWindowSize, List preFilterQueryBuilders) { super(childRetrievers, rankWindowSize); + this.preFilterQueryBuilders = preFilterQueryBuilders; } @Override - protected TestCompoundRetrieverBuilder clone(List newChildRetrievers) { - return new TestCompoundRetrieverBuilder(newChildRetrievers, rankWindowSize); + protected TestCompoundRetrieverBuilder clone(List newChildRetrievers, List newPreFilterQueryBuilders) { + return new TestCompoundRetrieverBuilder(newChildRetrievers, rankWindowSize, newPreFilterQueryBuilders); } @Override diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilder.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilder.java index 54a89d061de35..5b27cc7a3e05a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilder.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilder.java @@ -110,12 +110,14 @@ public QueryRuleRetrieverBuilder( Map matchCriteria, List retrieverSource, int rankWindowSize, - String retrieverName + String retrieverName, + List preFilterQueryBuilders ) { super(retrieverSource, rankWindowSize); this.rulesetIds = rulesetIds; this.matchCriteria = matchCriteria; this.retrieverName = retrieverName; + this.preFilterQueryBuilders = preFilterQueryBuilders; } @Override @@ -156,8 +158,15 @@ public void doToXContent(XContentBuilder builder, Params params) throws IOExcept } @Override - protected QueryRuleRetrieverBuilder clone(List newChildRetrievers) { - return new QueryRuleRetrieverBuilder(rulesetIds, matchCriteria, newChildRetrievers, rankWindowSize, retrieverName); + protected QueryRuleRetrieverBuilder clone(List newChildRetrievers, List newPreFilterQueryBuilders) { + return new QueryRuleRetrieverBuilder( + rulesetIds, + matchCriteria, + newChildRetrievers, + rankWindowSize, + retrieverName, + newPreFilterQueryBuilders + ); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java index c239319b6283a..fd2427dc8ac6a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java @@ -129,7 +129,10 @@ public TextSimilarityRankRetrieverBuilder( } @Override - protected TextSimilarityRankRetrieverBuilder clone(List newChildRetrievers) { + protected TextSimilarityRankRetrieverBuilder clone( + List newChildRetrievers, + List newPreFilterQueryBuilders + ) { return new TextSimilarityRankRetrieverBuilder( newChildRetrievers, inferenceId, @@ -138,7 +141,7 @@ protected TextSimilarityRankRetrieverBuilder clone(List newChil rankWindowSize, minScore, retrieverName, - preFilterQueryBuilders + newPreFilterQueryBuilders ); } diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java index 37e1807d138aa..ae35153b6f39f 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java @@ -33,6 +33,7 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; import org.elasticsearch.search.vectors.QueryVectorBuilder; +import org.elasticsearch.search.vectors.TestQueryVectorBuilderPlugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.elasticsearch.xcontent.XContentBuilder; @@ -57,7 +58,6 @@ public class RRFRetrieverBuilderIT extends ESIntegTestCase { protected static String INDEX = "test_index"; - protected static final String ID_FIELD = "_id"; protected static final String DOC_FIELD = "doc"; protected static final String TEXT_FIELD = "text"; protected static final String VECTOR_FIELD = "vector"; @@ -743,6 +743,42 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder expectThrows(UnsupportedOperationException.class, () -> client().prepareSearch(INDEX).setSource(source).get()); } + public void testRRFFiltersPropagatedToKnnQueryVectorBuilder() { + final int rankWindowSize = 100; + final int rankConstant = 10; + SearchSourceBuilder source = new SearchSourceBuilder(); + // this will retriever all but 7 only due to top-level filter + StandardRetrieverBuilder standardRetriever = new StandardRetrieverBuilder(QueryBuilders.matchAllQuery()); + // this will too retrieve just doc 7 + KnnRetrieverBuilder knnRetriever = new KnnRetrieverBuilder( + "vector", + null, + new TestQueryVectorBuilderPlugin.TestQueryVectorBuilder(new float[] { 3 }), + 10, + 10, + null + ); + source.retriever( + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(standardRetriever, null), + new CompoundRetrieverBuilder.RetrieverSource(knnRetriever, null) + ), + rankWindowSize, + rankConstant + ) + ); + source.retriever().getPreFilterQueryBuilders().add(QueryBuilders.boolQuery().must(QueryBuilders.termQuery(DOC_FIELD, "doc_7"))); + source.size(10); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value(), equalTo(1L)); + assertThat(resp.getHits().getHits()[0].getId(), equalTo("doc_7")); + }); + } + public void testRewriteOnce() { final float[] vector = new float[] { 1 }; AtomicInteger numAsyncCalls = new AtomicInteger(); diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFFeatures.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFFeatures.java index bbc0f622724a3..bb61fa951948d 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFFeatures.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFFeatures.java @@ -12,6 +12,7 @@ import java.util.Set; +import static org.elasticsearch.search.retriever.CompoundRetrieverBuilder.INNER_RETRIEVERS_FILTER_SUPPORT; import static org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilder.RRF_RETRIEVER_COMPOSITION_SUPPORTED; /** @@ -23,4 +24,9 @@ public class RRFFeatures implements FeatureSpecification { public Set getFeatures() { return Set.of(RRFRetrieverBuilder.RRF_RETRIEVER_SUPPORTED, RRF_RETRIEVER_COMPOSITION_SUPPORTED); } + + @Override + public Set getTestFeatures() { + return Set.of(INNER_RETRIEVERS_FILTER_SUPPORT); + } } diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java index 792ff4eac3893..f1171b74f7468 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.util.Maps; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.search.rank.RankBuilder; import org.elasticsearch.search.rank.RankDoc; @@ -108,8 +109,10 @@ public String getName() { } @Override - protected RRFRetrieverBuilder clone(List newRetrievers) { - return new RRFRetrieverBuilder(newRetrievers, this.rankWindowSize, this.rankConstant); + protected RRFRetrieverBuilder clone(List newRetrievers, List newPreFilterQueryBuilders) { + RRFRetrieverBuilder clone = new RRFRetrieverBuilder(newRetrievers, this.rankWindowSize, this.rankConstant); + clone.preFilterQueryBuilders = newPreFilterQueryBuilders; + return clone; } @Override diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml index 42c01f0b9636c..cb30542d80003 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml @@ -1071,3 +1071,77 @@ setup: - match: { hits.hits.2.inner_hits.nested_data_field.hits.total.value: 0 } - match: { hits.hits.2.inner_hits.nested_vector_field.hits.total.value: 0 } + + +--- +"rrf retriever with filters to be passed to nested rrf retrievers": + - requires: + cluster_features: 'inner_retrievers_filter_support' + reason: 'requires fix for properly propagating filters to nested sub-retrievers' + + - do: + search: + _source: false + index: test + body: + retriever: + { + rrf: + { + filter: { + term: { + keyword: "technology" + } + }, + retrievers: [ + { + rrf: { + retrievers: [ + { + # this should only return docs 3 and 5 due to top level filter + standard: { + query: { + knn: { + field: vector, + query_vector: [ 4.0 ], + k: 3 + } + } + } }, + { + # this should return no docs as no docs match both biology and technology + standard: { + query: { + term: { + keyword: "biology" + } + } + } + } + ], + rank_window_size: 10, + rank_constant: 10 + } + }, + # this should only return doc 5 + { + standard: { + query: { + term: { + text: "term5" + } + } + } + } + ], + rank_window_size: 10, + rank_constant: 10 + } + } + size: 10 + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "3" } + + From 6087badce32fbcd095d115d4aaa69b62bd967a26 Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Thu, 5 Dec 2024 08:21:39 +0100 Subject: [PATCH 036/204] cleanups (#117997) --- .../core/planner/ExpressionTranslators.java | 4 +- .../xpack/esql/core/util/CollectionUtils.java | 15 +----- .../xpack/ql/type/DataTypes.java | 50 ++++++------------- 3 files changed, 18 insertions(+), 51 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java index 7836522c77130..468d076c1b7ef 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java @@ -107,9 +107,7 @@ protected Query asQuery(Not not, TranslatorHandler handler) { } public static Query doTranslate(Not not, TranslatorHandler handler) { - Query wrappedQuery = handler.asQuery(not.field()); - Query q = wrappedQuery.negate(not.source()); - return q; + return handler.asQuery(not.field()).negate(not.source()); } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java index 8bfcf4ca5c405..ce0540687121f 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java @@ -30,12 +30,8 @@ public static List combine(List left, List righ } List list = new ArrayList<>(left.size() + right.size()); - if (left.isEmpty() == false) { - list.addAll(left); - } - if (right.isEmpty() == false) { - list.addAll(right); - } + list.addAll(left); + list.addAll(right); return list; } @@ -73,13 +69,6 @@ public static List combine(Collection left, T... entries) { return list; } - public static int mapSize(int size) { - if (size < 2) { - return size + 1; - } - return (int) (size / 0.75f + 1f); - } - @SafeVarargs @SuppressWarnings("varargs") public static List nullSafeList(T... entries) { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java index 6aa47f7c817a7..c67d943b11e22 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java @@ -112,41 +112,21 @@ public static DataType fromEs(String name) { } public static DataType fromJava(Object value) { - if (value == null) { - return NULL; - } - if (value instanceof Integer) { - return INTEGER; - } - if (value instanceof Long) { - return LONG; - } - if (value instanceof BigInteger) { - return UNSIGNED_LONG; - } - if (value instanceof Boolean) { - return BOOLEAN; - } - if (value instanceof Double) { - return DOUBLE; - } - if (value instanceof Float) { - return FLOAT; - } - if (value instanceof Byte) { - return BYTE; - } - if (value instanceof Short) { - return SHORT; - } - if (value instanceof ZonedDateTime) { - return DATETIME; - } - if (value instanceof String || value instanceof Character) { - return KEYWORD; - } - - return null; + return switch (value) { + case null -> NULL; + case Integer i -> INTEGER; + case Long l -> LONG; + case BigInteger bigInteger -> UNSIGNED_LONG; + case Boolean b -> BOOLEAN; + case Double v -> DOUBLE; + case Float v -> FLOAT; + case Byte b -> BYTE; + case Short s -> SHORT; + case ZonedDateTime zonedDateTime -> DATETIME; + case String s -> KEYWORD; + case Character c -> KEYWORD; + default -> null; + }; } public static boolean isUnsupported(DataType from) { From 2051401d5049aaa6308838920230514698321b38 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 5 Dec 2024 09:05:32 +0100 Subject: [PATCH 037/204] Address mapping and compute engine runtime field issues (#117792) This change addresses the following issues: Fields mapped as runtime fields not getting stored if source mode is synthetic. Address java.io.EOFException when an es|ql query uses multiple runtime fields that fallback to source when source mode is synthetic. (1) Address concurrency issue when runtime fields get pushed down to Lucene. (2) 1: ValueSourceOperator can read values in row striding or columnar fashion. When values are read in columnar fashion and multiple runtime fields synthetize source then this can cause the same SourceProvider evaluation the same range of docs ids multiple times. This can then result in unexpected io errors at the codec level. This is because the same doc value instances are used by SourceProvider. Re-evaluating the same docids is in violation of the contract of the DocIdSetIterator#advance(...) / DocIdSetIterator#advanceExact(...) methods, which documents that unexpected behaviour can occur if target docid is lower than current docid position. Note that this is only an issue for synthetic source loader and not for stored source loader. And not when executing in row stride fashion which sometimes happen in compute engine and always happen in _search api. 2: The concurrency issue that arrises with source provider if source operator executes in parallel with data portioning set to DOC. The same SourceProvider instance then gets access by multiple threads concurrently. SourceProviders implementations are not designed to handle concurrent access. Closes #117644 --- docs/changelog/117792.yaml | 6 + .../index/mapper/DocumentParser.java | 4 +- .../index/query/SearchExecutionContext.java | 10 +- .../search/lookup/SearchLookup.java | 12 ++ .../xpack/esql/action/EsqlActionIT.java | 39 ++++++ .../planner/EsPhysicalOperationProviders.java | 12 +- .../xpack/esql/plugin/ComputeService.java | 17 ++- .../plugin/ReinitializingSourceProvider.java | 43 +++++++ .../xpack/logsdb/LogsdbRestIT.java | 117 ++++++++++++++++++ 9 files changed, 250 insertions(+), 10 deletions(-) create mode 100644 docs/changelog/117792.yaml create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java diff --git a/docs/changelog/117792.yaml b/docs/changelog/117792.yaml new file mode 100644 index 0000000000000..2d7ddda1ace40 --- /dev/null +++ b/docs/changelog/117792.yaml @@ -0,0 +1,6 @@ +pr: 117792 +summary: Address mapping and compute engine runtime field issues +area: Mapping +type: bug +issues: + - 117644 diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index e00e7b2320000..9ddb6f0d496a0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -946,7 +946,9 @@ public Query termQuery(Object value, SearchExecutionContext context) { protected void parseCreateField(DocumentParserContext context) { // Run-time fields are mapped to this mapper, so it needs to handle storing values for use in synthetic source. // #parseValue calls this method once the run-time field is created. - if (context.dynamic() == ObjectMapper.Dynamic.RUNTIME && context.canAddIgnoredField()) { + var fieldType = context.mappingLookup().getFieldType(path); + boolean isRuntimeField = fieldType instanceof AbstractScriptFieldType; + if ((context.dynamic() == ObjectMapper.Dynamic.RUNTIME || isRuntimeField) && context.canAddIgnoredField()) { try { context.addIgnoredField( IgnoredSourceFieldMapper.NameValue.fromContext(context, path, context.encodeFlattenedToken()) diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index b07112440d3c2..d5e48a6a54daa 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -493,14 +493,18 @@ public boolean containsBrokenAnalysis(String field) { */ public SearchLookup lookup() { if (this.lookup == null) { - SourceProvider sourceProvider = isSourceSynthetic() - ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()) - : SourceProvider.fromStoredFields(); + var sourceProvider = createSourceProvider(); setLookupProviders(sourceProvider, LeafFieldLookupProvider.fromStoredFields()); } return this.lookup; } + public SourceProvider createSourceProvider() { + return isSourceSynthetic() + ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()) + : SourceProvider.fromStoredFields(); + } + /** * Replace the standard source provider and field lookup provider on the SearchLookup * diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index f7f8cee30ee15..9eb0170af5efb 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -102,6 +102,14 @@ private SearchLookup(SearchLookup searchLookup, Set fieldChain) { this.fieldLookupProvider = searchLookup.fieldLookupProvider; } + private SearchLookup(SearchLookup searchLookup, SourceProvider sourceProvider, Set fieldChain) { + this.fieldChain = Collections.unmodifiableSet(fieldChain); + this.sourceProvider = sourceProvider; + this.fieldTypeLookup = searchLookup.fieldTypeLookup; + this.fieldDataLookup = searchLookup.fieldDataLookup; + this.fieldLookupProvider = searchLookup.fieldLookupProvider; + } + /** * Creates a copy of the current {@link SearchLookup} that looks fields up in the same way, but also tracks field references * in order to detect cycles and prevent resolving fields that depend on more than {@link #MAX_FIELD_CHAIN_DEPTH} other fields. @@ -144,4 +152,8 @@ public IndexFieldData getForField(MappedFieldType fieldType, MappedFieldType. public Source getSource(LeafReaderContext ctx, int doc) throws IOException { return sourceProvider.getSource(ctx, doc); } + + public SearchLookup swapSourceProvider(SourceProvider sourceProvider) { + return new SearchLookup(this, sourceProvider, fieldChain); + } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index 147b13b36c44b..00f53d31165b1 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -18,6 +18,7 @@ import org.elasticsearch.client.internal.ClusterAdminClient; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; @@ -1648,6 +1649,44 @@ public void testMaxTruncationSizeSetting() { } } + public void testScriptField() throws Exception { + XContentBuilder mapping = JsonXContent.contentBuilder(); + mapping.startObject(); + { + mapping.startObject("runtime"); + { + mapping.startObject("k1"); + mapping.field("type", "long"); + mapping.endObject(); + mapping.startObject("k2"); + mapping.field("type", "long"); + mapping.endObject(); + } + mapping.endObject(); + { + mapping.startObject("properties"); + mapping.startObject("meter").field("type", "double").endObject(); + mapping.endObject(); + } + } + mapping.endObject(); + String sourceMode = randomBoolean() ? "stored" : "synthetic"; + Settings.Builder settings = indexSettings(1, 0).put(indexSettings()).put("index.mapping.source.mode", sourceMode); + client().admin().indices().prepareCreate("test-script").setMapping(mapping).setSettings(settings).get(); + for (int i = 0; i < 10; i++) { + index("test-script", Integer.toString(i), Map.of("k1", i, "k2", "b-" + i, "meter", 10000 * i)); + } + refresh("test-script"); + try (EsqlQueryResponse resp = run("FROM test-script | SORT k1 | LIMIT 10")) { + List k1Column = Iterators.toList(resp.column(0)); + assertThat(k1Column, contains(0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L)); + List k2Column = Iterators.toList(resp.column(1)); + assertThat(k2Column, contains(null, null, null, null, null, null, null, null, null, null)); + List meterColumn = Iterators.toList(resp.column(2)); + assertThat(meterColumn, contains(0.0, 10000.0, 20000.0, 30000.0, 40000.0, 50000.0, 60000.0, 70000.0, 80000.0, 90000.0)); + } + } + private void clearPersistentSettings(Setting... settings) { Settings.Builder clearedSettings = Settings.builder(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 7bf7d0e2d08eb..39e2a3bc1d5af 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -39,6 +39,7 @@ import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -348,7 +349,16 @@ public MappedFieldType.FieldExtractPreference fieldExtractPreference() { @Override public SearchLookup lookup() { - return ctx.lookup(); + boolean syntheticSource = SourceFieldMapper.isSynthetic(indexSettings()); + var searchLookup = ctx.lookup(); + if (syntheticSource) { + // in the context of scripts and when synthetic source is used the search lookup can't always be reused between + // users of SearchLookup. This is only an issue when scripts fallback to _source, but since we can't always + // accurately determine whether a script uses _source, we should do this for all script usages. + // This lookup() method is only invoked for scripts / runtime fields, so it is ok to do here. + searchLookup = searchLookup.swapSourceProvider(ctx.createSourceProvider()); + } + return searchLookup; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index c9c8635a60f57..ed037d24139f8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -45,6 +45,7 @@ import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; @@ -87,6 +88,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import static org.elasticsearch.xpack.esql.plugin.EsqlPlugin.ESQL_WORKER_THREAD_POOL_NAME; @@ -471,12 +473,17 @@ void runCompute(CancellableTask task, ComputeContext context, PhysicalPlan plan, List contexts = new ArrayList<>(context.searchContexts.size()); for (int i = 0; i < context.searchContexts.size(); i++) { SearchContext searchContext = context.searchContexts.get(i); + var searchExecutionContext = new SearchExecutionContext(searchContext.getSearchExecutionContext()) { + + @Override + public SourceProvider createSourceProvider() { + final Supplier supplier = () -> super.createSourceProvider(); + return new ReinitializingSourceProvider(supplier); + + } + }; contexts.add( - new EsPhysicalOperationProviders.DefaultShardContext( - i, - searchContext.getSearchExecutionContext(), - searchContext.request().getAliasFilter() - ) + new EsPhysicalOperationProviders.DefaultShardContext(i, searchExecutionContext, searchContext.request().getAliasFilter()) ); } final List drivers; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java new file mode 100644 index 0000000000000..b6b2c6dfec755 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plugin; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.search.lookup.SourceProvider; + +import java.io.IOException; +import java.util.function.Supplier; + +/** + * This is a workaround for when compute engine executes concurrently with data partitioning by docid. + */ +final class ReinitializingSourceProvider implements SourceProvider { + + private PerThreadSourceProvider perThreadProvider; + private final Supplier sourceProviderFactory; + + ReinitializingSourceProvider(Supplier sourceProviderFactory) { + this.sourceProviderFactory = sourceProviderFactory; + } + + @Override + public Source getSource(LeafReaderContext ctx, int doc) throws IOException { + var currentThread = Thread.currentThread(); + PerThreadSourceProvider provider = perThreadProvider; + if (provider == null || provider.creatingThread != currentThread) { + provider = new PerThreadSourceProvider(sourceProviderFactory.get(), currentThread); + this.perThreadProvider = provider; + } + return perThreadProvider.source.getSource(ctx, doc); + } + + private record PerThreadSourceProvider(SourceProvider source, Thread creatingThread) { + + } +} diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java index 2bf8b00cf551c..ef9480681f559 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java @@ -10,6 +10,8 @@ import org.elasticsearch.client.Request; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; @@ -17,6 +19,7 @@ import org.junit.ClassRule; import java.io.IOException; +import java.time.Instant; import java.util.List; import java.util.Map; @@ -108,4 +111,118 @@ public void testLogsdbSourceModeForLogsIndex() throws IOException { assertNull(settings.get("index.mapping.source.mode")); } + public void testEsqlRuntimeFields() throws IOException { + String mappings = """ + { + "runtime": { + "message_length": { + "type": "long" + }, + "log.offset": { + "type": "long" + } + }, + "dynamic": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "log" : { + "properties": { + "level": { + "type": "keyword" + }, + "file": { + "type": "keyword" + } + } + } + } + } + """; + String indexName = "test-foo"; + createIndex(indexName, Settings.builder().put("index.mode", "logsdb").build(), mappings); + + int numDocs = 500; + var sb = new StringBuilder(); + var now = Instant.now(); + + var expectedMinTimestamp = now; + for (int i = 0; i < numDocs; i++) { + String level = randomBoolean() ? "info" : randomBoolean() ? "warning" : randomBoolean() ? "error" : "fatal"; + String msg = randomAlphaOfLength(20); + String path = randomAlphaOfLength(8); + String messageLength = Integer.toString(msg.length()); + String offset = Integer.toString(randomNonNegativeInt()); + sb.append("{ \"create\": {} }").append('\n'); + if (randomBoolean()) { + sb.append( + """ + {"@timestamp":"$now","message":"$msg","message_length":$l,"file":{"level":"$level","offset":5,"file":"$path"}} + """.replace("$now", formatInstant(now)) + .replace("$level", level) + .replace("$msg", msg) + .replace("$path", path) + .replace("$l", messageLength) + .replace("$o", offset) + ); + } else { + sb.append(""" + {"@timestamp": "$now", "message": "$msg", "message_length": $l} + """.replace("$now", formatInstant(now)).replace("$msg", msg).replace("$l", messageLength)); + } + sb.append('\n'); + if (i != numDocs - 1) { + now = now.plusSeconds(1); + } + } + var expectedMaxTimestamp = now; + + var bulkRequest = new Request("POST", "/" + indexName + "/_bulk"); + bulkRequest.setJsonEntity(sb.toString()); + bulkRequest.addParameter("refresh", "true"); + var bulkResponse = client().performRequest(bulkRequest); + var bulkResponseBody = responseAsMap(bulkResponse); + assertThat(bulkResponseBody, Matchers.hasEntry("errors", false)); + + var forceMergeRequest = new Request("POST", "/" + indexName + "/_forcemerge"); + forceMergeRequest.addParameter("max_num_segments", "1"); + var forceMergeResponse = client().performRequest(forceMergeRequest); + assertOK(forceMergeResponse); + + String query = "FROM test-foo | STATS count(*), min(@timestamp), max(@timestamp), min(message_length), max(message_length)" + + " ,sum(message_length), avg(message_length), min(log.offset), max(log.offset) | LIMIT 1"; + final Request esqlRequest = new Request("POST", "/_query"); + esqlRequest.setJsonEntity(""" + { + "query": "$query" + } + """.replace("$query", query)); + var esqlResponse = client().performRequest(esqlRequest); + assertOK(esqlResponse); + Map esqlResponseBody = responseAsMap(esqlResponse); + + List values = (List) esqlResponseBody.get("values"); + assertThat(values, Matchers.not(Matchers.empty())); + var count = ((List) values.getFirst()).get(0); + assertThat(count, equalTo(numDocs)); + logger.warn("VALUES: {}", values); + + var minTimestamp = ((List) values.getFirst()).get(1); + assertThat(minTimestamp, equalTo(formatInstant(expectedMinTimestamp))); + var maxTimestamp = ((List) values.getFirst()).get(2); + assertThat(maxTimestamp, equalTo(formatInstant(expectedMaxTimestamp))); + + var minLength = ((List) values.getFirst()).get(3); + assertThat(minLength, equalTo(20)); + var maxLength = ((List) values.getFirst()).get(4); + assertThat(maxLength, equalTo(20)); + var sumLength = ((List) values.getFirst()).get(5); + assertThat(sumLength, equalTo(20 * numDocs)); + } + + static String formatInstant(Instant instant) { + return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); + } + } From 6fe8894adce71905cea578d41d652cb88ec3c6e8 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 5 Dec 2024 08:59:48 +0000 Subject: [PATCH 038/204] Collapse transport versions for 8.16 (#117991) --- .../geoip/EnterpriseGeoIpTaskState.java | 2 +- .../ingest/geoip/GeoIpTaskState.java | 2 +- .../ingest/geoip/IngestGeoIpMetadata.java | 4 +- .../geoip/direct/DatabaseConfiguration.java | 4 +- .../elasticsearch/ElasticsearchException.java | 4 +- .../org/elasticsearch/TransportVersions.java | 73 +------------------ .../TransportGetAllocationStatsAction.java | 4 +- .../stats/NodesStatsRequestParameters.java | 4 +- .../create/CreateSnapshotRequest.java | 4 +- .../stats/ClusterStatsNodeResponse.java | 13 +--- .../stats/RemoteClusterStatsRequest.java | 4 +- .../admin/cluster/stats/SearchUsageStats.java | 6 +- .../stats/TransportClusterStatsAction.java | 5 +- .../get/GetComponentTemplateAction.java | 6 +- .../get/GetComposableIndexTemplateAction.java | 6 +- .../post/SimulateIndexTemplateResponse.java | 6 +- .../action/bulk/BulkItemResponse.java | 4 +- .../action/bulk/BulkRequest.java | 4 +- .../action/bulk/BulkResponse.java | 4 +- .../bulk/IndexDocFailureStoreStatus.java | 4 +- .../action/bulk/SimulateBulkRequest.java | 11 +-- .../datastreams/GetDataStreamAction.java | 8 +- .../FieldCapabilitiesIndexResponse.java | 8 +- .../action/index/IndexRequest.java | 8 +- .../action/index/IndexResponse.java | 8 +- .../action/search/OpenPointInTimeRequest.java | 4 +- .../search/OpenPointInTimeResponse.java | 2 +- .../action/search/SearchContextId.java | 6 +- .../action/search/SearchContextIdForNode.java | 8 +- .../TransportOpenPointInTimeAction.java | 3 +- .../action/support/IndicesOptions.java | 10 +-- .../cluster/health/ClusterIndexHealth.java | 4 +- .../cluster/health/ClusterShardHealth.java | 4 +- .../cluster/health/ClusterStateHealth.java | 4 +- .../cluster/metadata/DataStream.java | 11 ++- .../metadata/InferenceFieldMetadata.java | 7 +- .../cluster/routing/RoutingTable.java | 8 +- .../common/io/stream/StreamInput.java | 7 +- .../common/io/stream/StreamOutput.java | 9 +-- .../index/engine/CommitStats.java | 4 +- .../index/mapper/NodeMappingStats.java | 4 +- .../index/query/IntervalsSourceProvider.java | 4 +- .../index/query/RankDocsQueryBuilder.java | 8 +- .../index/search/stats/SearchStats.java | 4 +- .../inference/EmptySecretSettings.java | 2 +- .../inference/ModelConfigurations.java | 4 +- .../ingest/EnterpriseGeoIpTask.java | 2 +- .../elasticsearch/search/DocValueFormat.java | 4 +- .../elasticsearch/search/rank/RankDoc.java | 2 +- .../search/vectors/ExactKnnQueryBuilder.java | 6 +- .../vectors/KnnScoreDocQueryBuilder.java | 6 +- .../snapshots/RegisteredPolicySnapshots.java | 4 +- .../elasticsearch/TransportVersionTests.java | 2 +- .../NodesStatsRequestParametersTests.java | 2 +- .../cluster/stats/SearchUsageStatsTests.java | 2 +- .../common/io/stream/AbstractStreamTests.java | 15 ++-- .../DataStreamLifecycleFeatureSetUsage.java | 4 +- .../core/enrich/action/EnrichStatsAction.java | 4 +- .../ilm/IndexLifecycleExplainResponse.java | 4 +- .../core/ilm/SearchableSnapshotAction.java | 8 +- .../action/DeleteInferenceEndpointAction.java | 4 +- .../action/GetInferenceModelAction.java | 4 +- .../ml/MachineLearningFeatureSetUsage.java | 4 +- .../CreateTrainedModelAssignmentAction.java | 4 +- .../StartTrainedModelDeploymentAction.java | 8 +- .../UpdateTrainedModelDeploymentAction.java | 4 +- .../core/ml/calendars/ScheduledEvent.java | 4 +- .../inference/assignment/AssignmentStats.java | 4 +- .../assignment/TrainedModelAssignment.java | 4 +- .../trainedmodel/LearningToRankConfig.java | 3 +- .../core/ml/job/config/DetectionRule.java | 4 +- .../ConfigurableClusterPrivileges.java | 2 +- .../MachineLearningFeatureSetUsageTests.java | 2 +- .../rules/QueryRulesetListItem.java | 6 +- ...setsActionResponseBWCSerializingTests.java | 3 +- ...lesetActionRequestBWCSerializingTests.java | 2 +- ...esetActionResponseBWCSerializingTests.java | 2 +- .../xpack/esql/core/type/EsField.java | 2 +- .../xpack/esql/core/util/PlanStreamInput.java | 2 +- .../esql/core/util/PlanStreamOutput.java | 2 +- .../compute/operator/AggregationOperator.java | 4 +- .../compute/operator/DriverProfile.java | 4 +- .../compute/operator/DriverSleeps.java | 4 +- .../xpack/esql/action/EsqlExecutionInfo.java | 4 +- .../xpack/esql/action/EsqlQueryResponse.java | 4 +- .../esql/action/EsqlResolveFieldsAction.java | 2 +- .../esql/enrich/ResolvedEnrichPolicy.java | 6 +- .../function/UnsupportedAttribute.java | 6 +- .../function/aggregate/AggregateFunction.java | 8 +- .../function/aggregate/CountDistinct.java | 6 +- .../function/aggregate/FromPartial.java | 6 +- .../function/aggregate/Percentile.java | 6 +- .../expression/function/aggregate/Rate.java | 6 +- .../function/aggregate/ToPartial.java | 6 +- .../expression/function/aggregate/Top.java | 6 +- .../function/aggregate/WeightedAvg.java | 6 +- .../xpack/esql/index/EsIndex.java | 4 +- .../xpack/esql/io/stream/PlanStreamInput.java | 6 +- .../esql/io/stream/PlanStreamOutput.java | 6 +- .../esql/plan/physical/AggregateExec.java | 2 +- .../xpack/esql/plugin/ComputeResponse.java | 4 +- .../xpack/esql/plugin/DataNodeRequest.java | 4 +- .../xpack/esql/plugin/RemoteClusterPlan.java | 4 +- .../esql/querydsl/query/SingleValueQuery.java | 4 +- .../xpack/esql/session/Configuration.java | 4 +- .../esql/plugin/ClusterRequestTests.java | 6 +- .../SentenceBoundaryChunkingSettings.java | 6 +- .../WordBoundaryChunkingSettings.java | 2 +- .../rank/random/RandomRankBuilder.java | 2 +- .../textsimilarity/TextSimilarityRankDoc.java | 2 +- .../AlibabaCloudSearchService.java | 2 +- .../AlibabaCloudSearchServiceSettings.java | 2 +- ...aCloudSearchCompletionServiceSettings.java | 2 +- ...babaCloudSearchCompletionTaskSettings.java | 2 +- ...aCloudSearchEmbeddingsServiceSettings.java | 2 +- ...babaCloudSearchEmbeddingsTaskSettings.java | 2 +- ...ibabaCloudSearchRerankServiceSettings.java | 2 +- .../AlibabaCloudSearchRerankTaskSettings.java | 2 +- ...ibabaCloudSearchSparseServiceSettings.java | 2 +- .../AlibabaCloudSearchSparseTaskSettings.java | 2 +- .../rerank/CohereRerankServiceSettings.java | 4 +- .../elastic/ElasticInferenceService.java | 2 +- ...erviceSparseEmbeddingsServiceSettings.java | 2 +- .../ElasticsearchInternalServiceSettings.java | 14 ++-- .../ibmwatsonx/IbmWatsonxService.java | 2 +- .../IbmWatsonxEmbeddingsServiceSettings.java | 2 +- .../ltr/LearningToRankRescorerBuilder.java | 2 +- .../xpack/rank/rrf/RRFRankDoc.java | 6 +- .../xpack/security/authc/ApiKeyService.java | 8 +- .../authz/store/NativeRolesStore.java | 4 +- .../RolesBackwardsCompatibilityIT.java | 10 +-- 131 files changed, 260 insertions(+), 414 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java index c4d0aef0183ed..c128af69009be 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java @@ -123,7 +123,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER; + return TransportVersions.V_8_16_0; } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java index 47ca79e3cb3b9..96525d427d3e8 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java @@ -44,7 +44,7 @@ public class GeoIpTaskState implements PersistentTaskState, VersionedNamedWriteable { private static boolean includeSha256(TransportVersion version) { - return version.isPatchFrom(TransportVersions.V_8_15_0) || version.onOrAfter(TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER); + return version.onOrAfter(TransportVersions.V_8_15_0); } private static final ParseField DATABASES = new ParseField("databases"); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java index b6e73f3f33f7c..a50fe7dee9008 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java @@ -69,7 +69,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER; + return TransportVersions.V_8_16_0; } public Map getDatabases() { @@ -138,7 +138,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER; + return TransportVersions.V_8_16_0; } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java index a26364f9305e1..aa48c73cf1d73 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java @@ -138,7 +138,7 @@ public DatabaseConfiguration(StreamInput in) throws IOException { } private static Provider readProvider(StreamInput in) throws IOException { - if (in.getTransportVersion().onOrAfter(TransportVersions.INGEST_GEO_DATABASE_PROVIDERS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { return in.readNamedWriteable(Provider.class); } else { // prior to the above version, everything was always a maxmind, so this half of the if is logical @@ -154,7 +154,7 @@ public static DatabaseConfiguration parse(XContentParser parser, String id) { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeString(name); - if (out.getTransportVersion().onOrAfter(TransportVersions.INGEST_GEO_DATABASE_PROVIDERS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeNamedWriteable(provider); } else { if (provider instanceof Maxmind maxmind) { diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java index 3c5c365654206..fcb5c20c28162 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java @@ -1947,13 +1947,13 @@ private enum ElasticsearchExceptionHandle { org.elasticsearch.ingest.IngestPipelineException.class, org.elasticsearch.ingest.IngestPipelineException::new, 182, - TransportVersions.INGEST_PIPELINE_EXCEPTION_ADDED + TransportVersions.V_8_16_0 ), INDEX_RESPONSE_WRAPPER_EXCEPTION( IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus.class, IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus::new, 183, - TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE + TransportVersions.V_8_16_0 ); final Class exceptionClass; diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 2e4842912dfae..1a1219825bbbe 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -104,78 +104,7 @@ static TransportVersion def(int id) { public static final TransportVersion V_8_14_0 = def(8_636_00_1); public static final TransportVersion V_8_15_0 = def(8_702_00_2); public static final TransportVersion V_8_15_2 = def(8_702_00_3); - public static final TransportVersion QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_15 = def(8_702_00_4); - public static final TransportVersion ML_INFERENCE_DONT_DELETE_WHEN_SEMANTIC_TEXT_EXISTS = def(8_703_00_0); - public static final TransportVersion INFERENCE_ADAPTIVE_ALLOCATIONS = def(8_704_00_0); - public static final TransportVersion INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN = def(8_705_00_0); - public static final TransportVersion ML_INFERENCE_COHERE_UNUSED_RERANK_SETTINGS_REMOVED = def(8_706_00_0); - public static final TransportVersion ENRICH_CACHE_STATS_SIZE_ADDED = def(8_707_00_0); - public static final TransportVersion ENTERPRISE_GEOIP_DOWNLOADER = def(8_708_00_0); - public static final TransportVersion NODES_STATS_ENUM_SET = def(8_709_00_0); - public static final TransportVersion MASTER_NODE_METRICS = def(8_710_00_0); - public static final TransportVersion SEGMENT_LEVEL_FIELDS_STATS = def(8_711_00_0); - public static final TransportVersion ML_ADD_DETECTION_RULE_PARAMS = def(8_712_00_0); - public static final TransportVersion FIX_VECTOR_SIMILARITY_INNER_HITS = def(8_713_00_0); - public static final TransportVersion INDEX_REQUEST_UPDATE_BY_DOC_ORIGIN = def(8_714_00_0); - public static final TransportVersion ESQL_ATTRIBUTE_CACHED_SERIALIZATION = def(8_715_00_0); - public static final TransportVersion REGISTER_SLM_STATS = def(8_716_00_0); - public static final TransportVersion ESQL_NESTED_UNSUPPORTED = def(8_717_00_0); - public static final TransportVersion ESQL_SINGLE_VALUE_QUERY_SOURCE = def(8_718_00_0); - public static final TransportVersion ESQL_ORIGINAL_INDICES = def(8_719_00_0); - public static final TransportVersion ML_INFERENCE_EIS_INTEGRATION_ADDED = def(8_720_00_0); - public static final TransportVersion INGEST_PIPELINE_EXCEPTION_ADDED = def(8_721_00_0); - public static final TransportVersion ZDT_NANOS_SUPPORT_BROKEN = def(8_722_00_0); - public static final TransportVersion REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES = def(8_723_00_0); - public static final TransportVersion RANDOM_RERANKER_RETRIEVER = def(8_724_00_0); - public static final TransportVersion ESQL_PROFILE_SLEEPS = def(8_725_00_0); - public static final TransportVersion ZDT_NANOS_SUPPORT = def(8_726_00_0); - public static final TransportVersion LTR_SERVERLESS_RELEASE = def(8_727_00_0); - public static final TransportVersion ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT = def(8_728_00_0); - public static final TransportVersion RANK_DOCS_RETRIEVER = def(8_729_00_0); - public static final TransportVersion ESQL_ES_FIELD_CACHED_SERIALIZATION = def(8_730_00_0); - public static final TransportVersion ADD_MANAGE_ROLES_PRIVILEGE = def(8_731_00_0); - public static final TransportVersion REPOSITORIES_TELEMETRY = def(8_732_00_0); - public static final TransportVersion ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED = def(8_733_00_0); - public static final TransportVersion FIELD_CAPS_RESPONSE_INDEX_MODE = def(8_734_00_0); - public static final TransportVersion GET_DATA_STREAMS_VERBOSE = def(8_735_00_0); - public static final TransportVersion ESQL_ADD_INDEX_MODE_CONCRETE_INDICES = def(8_736_00_0); - public static final TransportVersion UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH = def(8_737_00_0); - public static final TransportVersion ESQL_AGGREGATE_EXEC_TRACKS_INTERMEDIATE_ATTRS = def(8_738_00_0); - public static final TransportVersion CCS_TELEMETRY_STATS = def(8_739_00_0); - public static final TransportVersion GLOBAL_RETENTION_TELEMETRY = def(8_740_00_0); - public static final TransportVersion ROUTING_TABLE_VERSION_REMOVED = def(8_741_00_0); - public static final TransportVersion ML_SCHEDULED_EVENT_TIME_SHIFT_CONFIGURATION = def(8_742_00_0); - public static final TransportVersion SIMULATE_COMPONENT_TEMPLATES_SUBSTITUTIONS = def(8_743_00_0); - public static final TransportVersion ML_INFERENCE_IBM_WATSONX_EMBEDDINGS_ADDED = def(8_744_00_0); - public static final TransportVersion BULK_INCREMENTAL_STATE = def(8_745_00_0); - public static final TransportVersion FAILURE_STORE_STATUS_IN_INDEX_RESPONSE = def(8_746_00_0); - public static final TransportVersion ESQL_AGGREGATION_OPERATOR_STATUS_FINISH_NANOS = def(8_747_00_0); - public static final TransportVersion ML_TELEMETRY_MEMORY_ADDED = def(8_748_00_0); - public static final TransportVersion ILM_ADD_SEARCHABLE_SNAPSHOT_TOTAL_SHARDS_PER_NODE = def(8_749_00_0); - public static final TransportVersion SEMANTIC_TEXT_SEARCH_INFERENCE_ID = def(8_750_00_0); - public static final TransportVersion ML_INFERENCE_CHUNKING_SETTINGS = def(8_751_00_0); - public static final TransportVersion SEMANTIC_QUERY_INNER_HITS = def(8_752_00_0); - public static final TransportVersion RETAIN_ILM_STEP_INFO = def(8_753_00_0); - public static final TransportVersion ADD_DATA_STREAM_OPTIONS = def(8_754_00_0); - public static final TransportVersion CCS_REMOTE_TELEMETRY_STATS = def(8_755_00_0); - public static final TransportVersion ESQL_CCS_EXECUTION_INFO = def(8_756_00_0); - public static final TransportVersion REGEX_AND_RANGE_INTERVAL_QUERIES = def(8_757_00_0); - public static final TransportVersion RRF_QUERY_REWRITE = def(8_758_00_0); - public static final TransportVersion SEARCH_FAILURE_STATS = def(8_759_00_0); - public static final TransportVersion INGEST_GEO_DATABASE_PROVIDERS = def(8_760_00_0); - public static final TransportVersion DATE_TIME_DOC_VALUES_LOCALES = def(8_761_00_0); - public static final TransportVersion FAST_REFRESH_RCO = def(8_762_00_0); - public static final TransportVersion TEXT_SIMILARITY_RERANKER_QUERY_REWRITE = def(8_763_00_0); - public static final TransportVersion SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS = def(8_764_00_0); - public static final TransportVersion RETRIEVERS_TELEMETRY_ADDED = def(8_765_00_0); - public static final TransportVersion ESQL_CACHED_STRING_SERIALIZATION = def(8_766_00_0); - public static final TransportVersion CHUNK_SENTENCE_OVERLAP_SETTING_ADDED = def(8_767_00_0); - public static final TransportVersion OPT_IN_ESQL_CCS_EXECUTION_INFO = def(8_768_00_0); - public static final TransportVersion QUERY_RULE_TEST_API = def(8_769_00_0); - public static final TransportVersion ESQL_PER_AGGREGATE_FILTER = def(8_770_00_0); - public static final TransportVersion ML_INFERENCE_ATTACH_TO_EXISTSING_DEPLOYMENT = def(8_771_00_0); - public static final TransportVersion CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY = def(8_772_00_0); - public static final TransportVersion INFERENCE_DONT_PERSIST_ON_READ_BACKPORT_8_16 = def(8_772_00_1); + public static final TransportVersion V_8_16_0 = def(8_772_00_1); public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO_BACKPORT_8_16 = def(8_772_00_2); public static final TransportVersion SKIP_INNER_HITS_SEARCH_SOURCE_BACKPORT_8_16 = def(8_772_00_3); public static final TransportVersion QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16 = def(8_772_00_4); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java index e14f229f17acf..d929fb457d5d1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java @@ -118,7 +118,7 @@ public Request(TimeValue masterNodeTimeout, TaskId parentTaskId, EnumSet public Request(StreamInput in) throws IOException { super(in); - this.metrics = in.getTransportVersion().onOrAfter(TransportVersions.MASTER_NODE_METRICS) + this.metrics = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readEnumSet(Metric.class) : EnumSet.of(Metric.ALLOCATIONS, Metric.FS); } @@ -127,7 +127,7 @@ public Request(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0); super.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.MASTER_NODE_METRICS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeEnumSet(metrics); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java index d34bc3ec0dc2f..c5e8f37ed3a96 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java @@ -117,7 +117,7 @@ public static Metric get(String name) { } public static void writeSetTo(StreamOutput out, EnumSet metrics) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.NODES_STATS_ENUM_SET)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeEnumSet(metrics); } else { out.writeCollection(metrics, (output, metric) -> output.writeString(metric.metricName)); @@ -125,7 +125,7 @@ public static void writeSetTo(StreamOutput out, EnumSet metrics) throws } public static EnumSet readSetFrom(StreamInput in) throws IOException { - if (in.getTransportVersion().onOrAfter(TransportVersions.NODES_STATS_ENUM_SET)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { return in.readEnumSet(Metric.class); } else { return in.readCollection((i) -> EnumSet.noneOf(Metric.class), (is, out) -> { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 9c9467db40de3..b6ced06623306 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -118,7 +118,7 @@ public CreateSnapshotRequest(StreamInput in) throws IOException { waitForCompletion = in.readBoolean(); partial = in.readBoolean(); userMetadata = in.readGenericMap(); - uuid = in.getTransportVersion().onOrAfter(TransportVersions.REGISTER_SLM_STATS) ? in.readOptionalString() : null; + uuid = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalString() : null; } @Override @@ -136,7 +136,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(waitForCompletion); out.writeBoolean(partial); out.writeGenericMap(userMetadata); - if (out.getTransportVersion().onOrAfter(TransportVersions.REGISTER_SLM_STATS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalString(uuid); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java index f99baa855404c..abeb73e5d8c3e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java @@ -44,14 +44,11 @@ public ClusterStatsNodeResponse(StreamInput in) throws IOException { } else { searchUsageStats = new SearchUsageStats(); } - if (in.getTransportVersion().onOrAfter(TransportVersions.REPOSITORIES_TELEMETRY)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { repositoryUsageStats = RepositoryUsageStats.readFrom(in); - } else { - repositoryUsageStats = RepositoryUsageStats.EMPTY; - } - if (in.getTransportVersion().onOrAfter(TransportVersions.CCS_TELEMETRY_STATS)) { ccsMetrics = new CCSTelemetrySnapshot(in); } else { + repositoryUsageStats = RepositoryUsageStats.EMPTY; ccsMetrics = new CCSTelemetrySnapshot(); } } @@ -118,12 +115,10 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_6_0)) { searchUsageStats.writeTo(out); } - if (out.getTransportVersion().onOrAfter(TransportVersions.REPOSITORIES_TELEMETRY)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { repositoryUsageStats.writeTo(out); - } // else just drop these stats, ok for bwc - if (out.getTransportVersion().onOrAfter(TransportVersions.CCS_TELEMETRY_STATS)) { ccsMetrics.writeTo(out); - } + } // else just drop these stats, ok for bwc } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsRequest.java index 47843a91351ee..6c3c5cbb50ece 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsRequest.java @@ -36,9 +36,9 @@ public ActionRequestValidationException validate() { @Override public void writeTo(StreamOutput out) throws IOException { - assert out.getTransportVersion().onOrAfter(TransportVersions.CCS_REMOTE_TELEMETRY_STATS) + assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) : "RemoteClusterStatsRequest is not supported by the remote cluster"; - if (out.getTransportVersion().before(TransportVersions.CCS_REMOTE_TELEMETRY_STATS)) { + if (out.getTransportVersion().before(TransportVersions.V_8_16_0)) { throw new UnsupportedOperationException("RemoteClusterStatsRequest is not supported by the remote cluster"); } super.writeTo(out); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStats.java index 0f6c56fd21bd7..a6e80b5efd08c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStats.java @@ -22,8 +22,8 @@ import java.util.Map; import java.util.Objects; -import static org.elasticsearch.TransportVersions.RETRIEVERS_TELEMETRY_ADDED; import static org.elasticsearch.TransportVersions.V_8_12_0; +import static org.elasticsearch.TransportVersions.V_8_16_0; /** * Holds a snapshot of the search usage statistics. @@ -71,7 +71,7 @@ public SearchUsageStats(StreamInput in) throws IOException { this.sections = in.readMap(StreamInput::readLong); this.totalSearchCount = in.readVLong(); this.rescorers = in.getTransportVersion().onOrAfter(V_8_12_0) ? in.readMap(StreamInput::readLong) : Map.of(); - this.retrievers = in.getTransportVersion().onOrAfter(RETRIEVERS_TELEMETRY_ADDED) ? in.readMap(StreamInput::readLong) : Map.of(); + this.retrievers = in.getTransportVersion().onOrAfter(V_8_16_0) ? in.readMap(StreamInput::readLong) : Map.of(); } @Override @@ -83,7 +83,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(V_8_12_0)) { out.writeMap(rescorers, StreamOutput::writeLong); } - if (out.getTransportVersion().onOrAfter(RETRIEVERS_TELEMETRY_ADDED)) { + if (out.getTransportVersion().onOrAfter(V_8_16_0)) { out.writeMap(retrievers, StreamOutput::writeLong); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 97585ea9a1024..2c20daa5d7afb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.store.AlreadyClosedException; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ActionType; @@ -72,8 +73,6 @@ import java.util.function.BooleanSupplier; import java.util.stream.Collectors; -import static org.elasticsearch.TransportVersions.CCS_REMOTE_TELEMETRY_STATS; - /** * Transport action implementing _cluster/stats API. */ @@ -450,7 +449,7 @@ protected void sendItemRequest(String clusterAlias, ActionListener { - if (connection.getTransportVersion().before(CCS_REMOTE_TELEMETRY_STATS)) { + if (connection.getTransportVersion().before(TransportVersions.V_8_16_0)) { responseListener.onResponse(null); } else { remoteClusterClient.execute(connection, TransportRemoteClusterStatsAction.REMOTE_TYPE, remoteRequest, responseListener); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java index c6d990e5a1d62..f729455edcc24 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java @@ -131,8 +131,7 @@ public Response(StreamInput in) throws IOException { } else { rolloverConfiguration = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - && in.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) { + if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { in.readOptionalWriteable(DataStreamGlobalRetention::read); } } @@ -190,8 +189,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - && out.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) { + if (out.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { out.writeOptionalWriteable(null); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java index a47f89030cc60..67f87476ea6a5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java @@ -132,8 +132,7 @@ public Response(StreamInput in) throws IOException { } else { rolloverConfiguration = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - && in.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) { + if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { in.readOptionalWriteable(DataStreamGlobalRetention::read); } } @@ -191,8 +190,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - && out.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) { + if (out.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { out.writeOptionalWriteable(null); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index 064c24cf4afa3..80e6fbfe051a4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -82,8 +82,7 @@ public SimulateIndexTemplateResponse(StreamInput in) throws IOException { rolloverConfiguration = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) ? in.readOptionalWriteable(RolloverConfiguration::new) : null; - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - && in.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) { + if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { in.readOptionalWriteable(DataStreamGlobalRetention::read); } } @@ -104,8 +103,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - && out.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) { + if (out.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { out.writeOptionalWriteable(null); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java index d5931c85bb2e1..1ff970de7525e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java @@ -200,7 +200,7 @@ public Failure(StreamInput in) throws IOException { seqNo = in.readZLong(); term = in.readVLong(); aborted = in.readBoolean(); - if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { failureStoreStatus = IndexDocFailureStoreStatus.read(in); } else { failureStoreStatus = IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; @@ -218,7 +218,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeZLong(seqNo); out.writeVLong(term); out.writeBoolean(aborted); - if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { failureStoreStatus.writeTo(out); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index f62b2f48fa2fd..91caebc420ffb 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -98,7 +98,7 @@ public BulkRequest(StreamInput in) throws IOException { for (DocWriteRequest request : requests) { indices.add(Objects.requireNonNull(request.index(), "request index must not be null")); } - if (in.getTransportVersion().onOrAfter(TransportVersions.BULK_INCREMENTAL_STATE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { incrementalState = new BulkRequest.IncrementalState(in); } else { incrementalState = BulkRequest.IncrementalState.EMPTY; @@ -454,7 +454,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(requests, DocWriteRequest::writeDocumentRequest); refreshPolicy.writeTo(out); out.writeTimeValue(timeout); - if (out.getTransportVersion().onOrAfter(TransportVersions.BULK_INCREMENTAL_STATE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { incrementalState.writeTo(out); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java index ec7a08007de93..12d3aa67ca9bb 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java @@ -46,7 +46,7 @@ public BulkResponse(StreamInput in) throws IOException { responses = in.readArray(BulkItemResponse::new, BulkItemResponse[]::new); tookInMillis = in.readVLong(); ingestTookInMillis = in.readZLong(); - if (in.getTransportVersion().onOrAfter(TransportVersions.BULK_INCREMENTAL_STATE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { incrementalState = new BulkRequest.IncrementalState(in); } else { incrementalState = BulkRequest.IncrementalState.EMPTY; @@ -151,7 +151,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeArray(responses); out.writeVLong(tookInMillis); out.writeZLong(ingestTookInMillis); - if (out.getTransportVersion().onOrAfter(TransportVersions.BULK_INCREMENTAL_STATE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { incrementalState.writeTo(out); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/IndexDocFailureStoreStatus.java b/server/src/main/java/org/elasticsearch/action/bulk/IndexDocFailureStoreStatus.java index cb83d693a415b..7367dfa1d53fd 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/IndexDocFailureStoreStatus.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/IndexDocFailureStoreStatus.java @@ -124,7 +124,7 @@ public ExceptionWithFailureStoreStatus(BulkItemResponse.Failure failure) { public ExceptionWithFailureStoreStatus(StreamInput in) throws IOException { super(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { failureStoreStatus = IndexDocFailureStoreStatus.fromId(in.readByte()); } else { failureStoreStatus = NOT_APPLICABLE_OR_UNKNOWN; @@ -134,7 +134,7 @@ public ExceptionWithFailureStoreStatus(StreamInput in) throws IOException { @Override protected void writeTo(StreamOutput out, Writer nestedExceptionsWriter) throws IOException { super.writeTo(out, nestedExceptionsWriter); - if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeByte(failureStoreStatus.getId()); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java index cc7fd431d8097..290d342e9dc12 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java @@ -135,14 +135,11 @@ public SimulateBulkRequest( public SimulateBulkRequest(StreamInput in) throws IOException { super(in); this.pipelineSubstitutions = (Map>) in.readGenericValue(); - if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_COMPONENT_TEMPLATES_SUBSTITUTIONS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.componentTemplateSubstitutions = (Map>) in.readGenericValue(); - } else { - componentTemplateSubstitutions = Map.of(); - } - if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS)) { this.indexTemplateSubstitutions = (Map>) in.readGenericValue(); } else { + componentTemplateSubstitutions = Map.of(); indexTemplateSubstitutions = Map.of(); } if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_MAPPING_ADDITION)) { @@ -156,10 +153,8 @@ public SimulateBulkRequest(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeGenericValue(pipelineSubstitutions); - if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_COMPONENT_TEMPLATES_SUBSTITUTIONS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeGenericValue(componentTemplateSubstitutions); - } - if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS)) { out.writeGenericValue(indexTemplateSubstitutions); } if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_MAPPING_ADDITION)) { diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index c1cf0fa7aab42..93c40ad18cc8a 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -112,7 +112,7 @@ public Request(StreamInput in) throws IOException { } else { this.includeDefaults = false; } - if (in.getTransportVersion().onOrAfter(TransportVersions.GET_DATA_STREAMS_VERBOSE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.verbose = in.readBoolean(); } else { this.verbose = false; @@ -127,7 +127,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeBoolean(includeDefaults); } - if (out.getTransportVersion().onOrAfter(TransportVersions.GET_DATA_STREAMS_VERBOSE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeBoolean(verbose); } } @@ -275,7 +275,7 @@ public DataStreamInfo( in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0) ? in.readOptionalWriteable(TimeSeries::new) : null, in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readMap(Index::new, IndexProperties::new) : Map.of(), in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readBoolean() : true, - in.getTransportVersion().onOrAfter(TransportVersions.GET_DATA_STREAMS_VERBOSE) ? in.readOptionalVLong() : null + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalVLong() : null ); } @@ -328,7 +328,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(indexSettingsValues); out.writeBoolean(templatePreferIlmValue); } - if (out.getTransportVersion().onOrAfter(TransportVersions.GET_DATA_STREAMS_VERBOSE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalVLong(maximumTimestamp); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java index d16100a64713e..6f510ad26f5ec 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java @@ -62,7 +62,7 @@ public FieldCapabilitiesIndexResponse( } else { this.indexMappingHash = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.indexMode = IndexMode.readFrom(in); } else { this.indexMode = IndexMode.STANDARD; @@ -77,7 +77,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(MAPPING_HASH_VERSION)) { out.writeOptionalString(indexMappingHash); } - if (out.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { IndexMode.writeTo(indexMode, out); } } @@ -105,7 +105,7 @@ static List readList(StreamInput input) throws I private static void collectCompressedResponses(StreamInput input, int groups, ArrayList responses) throws IOException { final CompressedGroup[] compressedGroups = new CompressedGroup[groups]; - final boolean readIndexMode = input.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE); + final boolean readIndexMode = input.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0); for (int i = 0; i < groups; i++) { final String[] indices = input.readStringArray(); final IndexMode indexMode = readIndexMode ? IndexMode.readFrom(input) : IndexMode.STANDARD; @@ -179,7 +179,7 @@ private static void writeCompressedResponses(StreamOutput output, Map { o.writeCollection(fieldCapabilitiesIndexResponses, (oo, r) -> oo.writeString(r.indexName)); var first = fieldCapabilitiesIndexResponses.get(0); - if (output.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE)) { + if (output.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { IndexMode.writeTo(first.indexMode, o); } o.writeString(first.indexMappingHash); diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index c0811e7424b0d..5254c6fd06db7 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -205,10 +205,8 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { in.readZLong(); // obsolete normalisedBytesParsed } - if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { in.readBoolean(); // obsolete originatesFromUpdateByScript - } - if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_DOC_ORIGIN)) { in.readBoolean(); // obsolete originatesFromUpdateByDoc } } @@ -789,10 +787,8 @@ private void writeBody(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { out.writeZLong(-1); // obsolete normalisedBytesParsed } - if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeBoolean(false); // obsolete originatesFromUpdateByScript - } - if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_DOC_ORIGIN)) { out.writeBoolean(false); // obsolete originatesFromUpdateByDoc } } diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java b/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java index 8d1bdf227e24d..7c45de8905174 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java @@ -46,7 +46,7 @@ public IndexResponse(ShardId shardId, StreamInput in) throws IOException { } else { executedPipelines = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { failureStoreStatus = IndexDocFailureStoreStatus.read(in); } else { failureStoreStatus = IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; @@ -60,7 +60,7 @@ public IndexResponse(StreamInput in) throws IOException { } else { executedPipelines = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { failureStoreStatus = IndexDocFailureStoreStatus.read(in); } else { failureStoreStatus = IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; @@ -126,7 +126,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeOptionalCollection(executedPipelines, StreamOutput::writeString); } - if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { failureStoreStatus.writeTo(out); } } @@ -137,7 +137,7 @@ public void writeThin(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeOptionalCollection(executedPipelines, StreamOutput::writeString); } - if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { failureStoreStatus.writeTo(out); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java index 969ba2ad983ce..d68e2ce1b02b7 100644 --- a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java @@ -63,7 +63,7 @@ public OpenPointInTimeRequest(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { this.indexFilter = in.readOptionalNamedWriteable(QueryBuilder.class); } - if (in.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.allowPartialSearchResults = in.readBoolean(); } } @@ -82,7 +82,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeOptionalWriteable(indexFilter); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeBoolean(allowPartialSearchResults); } else if (allowPartialSearchResults) { throw new IOException("[allow_partial_search_results] is not supported on nodes with version " + out.getTransportVersion()); diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java index 3c830c8ed9dc1..b3ffc564d848c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java @@ -47,7 +47,7 @@ public OpenPointInTimeResponse( @Override public void writeTo(StreamOutput out) throws IOException { out.writeBytesReference(pointInTimeId); - if (out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVInt(totalShards); out.writeVInt(successfulShards); out.writeVInt(failedShards); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java index ca810bb88653f..c2f1510341fb0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java @@ -63,14 +63,14 @@ public static BytesReference encode( TransportVersion version, ShardSearchFailure[] shardFailures ) { - assert shardFailures.length == 0 || version.onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT) + assert shardFailures.length == 0 || version.onOrAfter(TransportVersions.V_8_16_0) : "[allow_partial_search_results] cannot be enabled on a cluster that has not been fully upgraded to version [" - + TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT + + TransportVersions.V_8_16_0.toReleaseVersion() + "] or higher."; try (var out = new BytesStreamOutput()) { out.setTransportVersion(version); TransportVersion.writeVersion(version, out); - boolean allowNullContextId = out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT); + boolean allowNullContextId = out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0); int shardSize = searchPhaseResults.size() + (allowNullContextId ? shardFailures.length : 0); out.writeVInt(shardSize); for (var searchResult : searchPhaseResults) { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java index 7509a7b0fed04..f91a9d09f4bb4 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java @@ -37,7 +37,7 @@ public final class SearchContextIdForNode implements Writeable { } SearchContextIdForNode(StreamInput in) throws IOException { - boolean allowNull = in.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT); + boolean allowNull = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0); this.node = allowNull ? in.readOptionalString() : in.readString(); this.clusterAlias = in.readOptionalString(); this.searchContextId = allowNull ? in.readOptionalWriteable(ShardSearchContextId::new) : new ShardSearchContextId(in); @@ -45,7 +45,7 @@ public final class SearchContextIdForNode implements Writeable { @Override public void writeTo(StreamOutput out) throws IOException { - boolean allowNull = out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT); + boolean allowNull = out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0); if (allowNull) { out.writeOptionalString(node); } else { @@ -53,7 +53,7 @@ public void writeTo(StreamOutput out) throws IOException { // We should never set a null node if the cluster is not fully upgraded to a version that can handle it. throw new IOException( "Cannot write null node value to a node in version " - + out.getTransportVersion() + + out.getTransportVersion().toReleaseVersion() + ". The target node must be specified to retrieve the ShardSearchContextId." ); } @@ -67,7 +67,7 @@ public void writeTo(StreamOutput out) throws IOException { // We should never set a null search context id if the cluster is not fully upgraded to a version that can handle it. throw new IOException( "Cannot write null search context ID to a node in version " - + out.getTransportVersion() + + out.getTransportVersion().toReleaseVersion() + ". A valid search context ID is required to identify the shard's search context in this version." ); } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java index 9e60eedbad6a2..36ca0fba94372 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java @@ -104,8 +104,7 @@ public TransportOpenPointInTimeAction( protected void doExecute(Task task, OpenPointInTimeRequest request, ActionListener listener) { final ClusterState clusterState = clusterService.state(); // Check if all the nodes in this cluster know about the service - if (request.allowPartialSearchResults() - && clusterState.getMinTransportVersion().before(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) { + if (request.allowPartialSearchResults() && clusterState.getMinTransportVersion().before(TransportVersions.V_8_16_0)) { listener.onFailure( new ElasticsearchStatusException( format( diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java index 85889d8398cb1..ebbd47336e3da 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java @@ -982,12 +982,11 @@ public void writeIndicesOptions(StreamOutput out) throws IOException { states.add(WildcardStates.HIDDEN); } out.writeEnumSet(states); - if (out.getTransportVersion() - .between(TransportVersions.V_8_14_0, TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) { + if (out.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { out.writeBoolean(includeRegularIndices()); out.writeBoolean(includeFailureIndices()); } - if (out.getTransportVersion().onOrAfter(TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { selectorOptions.writeTo(out); } } @@ -1010,8 +1009,7 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti .ignoreThrottled(options.contains(Option.IGNORE_THROTTLED)) .build(); SelectorOptions selectorOptions = SelectorOptions.DEFAULT; - if (in.getTransportVersion() - .between(TransportVersions.V_8_14_0, TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) { + if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { // Reading from an older node, which will be sending two booleans that we must read out and ignore. var includeData = in.readBoolean(); var includeFailures = in.readBoolean(); @@ -1023,7 +1021,7 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti selectorOptions = SelectorOptions.FAILURES; } } - if (in.getTransportVersion().onOrAfter(TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { selectorOptions = SelectorOptions.read(in); } return new IndicesOptions( diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java index b6c1defe91a75..9cf567c219660 100644 --- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java +++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java @@ -111,7 +111,7 @@ public ClusterIndexHealth(final StreamInput in) throws IOException { unassignedShards = in.readVInt(); status = ClusterHealthStatus.readFrom(in); shards = in.readMapValues(ClusterShardHealth::new, ClusterShardHealth::getShardId); - if (in.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { unassignedPrimaryShards = in.readVInt(); } else { unassignedPrimaryShards = 0; @@ -203,7 +203,7 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeVInt(unassignedShards); out.writeByte(status.value()); out.writeMapValues(shards); - if (out.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVInt(unassignedPrimaryShards); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java index 63863542564cd..f512acb6e04d0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java +++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java @@ -96,7 +96,7 @@ public ClusterShardHealth(final StreamInput in) throws IOException { initializingShards = in.readVInt(); unassignedShards = in.readVInt(); primaryActive = in.readBoolean(); - if (in.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { unassignedPrimaryShards = in.readVInt(); } else { unassignedPrimaryShards = 0; @@ -167,7 +167,7 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeVInt(initializingShards); out.writeVInt(unassignedShards); out.writeBoolean(primaryActive); - if (out.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVInt(unassignedPrimaryShards); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java index 579429b5d51dd..31f275e29c368 100644 --- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java +++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java @@ -120,7 +120,7 @@ public ClusterStateHealth(final StreamInput in) throws IOException { status = ClusterHealthStatus.readFrom(in); indices = in.readMapValues(ClusterIndexHealth::new, ClusterIndexHealth::getIndex); activeShardsPercent = in.readDouble(); - if (in.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { unassignedPrimaryShards = in.readVInt(); } else { unassignedPrimaryShards = 0; @@ -212,7 +212,7 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeByte(status.value()); out.writeMapValues(indices); out.writeDouble(activeShardsPercent); - if (out.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVInt(unassignedPrimaryShards); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 4dcc7c73c280e..979434950cf7a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -71,6 +71,7 @@ public final class DataStream implements SimpleDiffable, ToXContentO public static final FeatureFlag FAILURE_STORE_FEATURE_FLAG = new FeatureFlag("failure_store"); public static final TransportVersion ADDED_FAILURE_STORE_TRANSPORT_VERSION = TransportVersions.V_8_12_0; public static final TransportVersion ADDED_AUTO_SHARDING_EVENT_VERSION = TransportVersions.V_8_14_0; + public static final TransportVersion ADD_DATA_STREAM_OPTIONS_VERSION = TransportVersions.V_8_16_0; public static boolean isFailureStoreFeatureFlagEnabled() { return FAILURE_STORE_FEATURE_FLAG.isEnabled(); @@ -200,9 +201,7 @@ public static DataStream read(StreamInput in) throws IOException { : null; // This boolean flag has been moved in data stream options var failureStoreEnabled = in.getTransportVersion() - .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, TransportVersions.ADD_DATA_STREAM_OPTIONS) - ? in.readBoolean() - : false; + .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, TransportVersions.V_8_16_0) ? in.readBoolean() : false; var failureIndices = in.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION) ? readIndices(in) : List.of(); @@ -216,7 +215,7 @@ public static DataStream read(StreamInput in) throws IOException { .setAutoShardingEvent(in.readOptionalWriteable(DataStreamAutoShardingEvent::new)); } DataStreamOptions dataStreamOptions; - if (in.getTransportVersion().onOrAfter(TransportVersions.ADD_DATA_STREAM_OPTIONS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { dataStreamOptions = in.readOptionalWriteable(DataStreamOptions::read); } else { // We cannot distinguish if failure store was explicitly disabled or not. Given that failure store @@ -1077,7 +1076,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(lifecycle); } if (out.getTransportVersion() - .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, TransportVersions.ADD_DATA_STREAM_OPTIONS)) { + .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, DataStream.ADD_DATA_STREAM_OPTIONS_VERSION)) { out.writeBoolean(isFailureStoreEnabled()); } if (out.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION)) { @@ -1093,7 +1092,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(failureIndices.rolloverOnWrite); out.writeOptionalWriteable(failureIndices.autoShardingEvent); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ADD_DATA_STREAM_OPTIONS)) { + if (out.getTransportVersion().onOrAfter(DataStream.ADD_DATA_STREAM_OPTIONS_VERSION)) { out.writeOptionalWriteable(dataStreamOptions.isEmpty() ? null : dataStreamOptions); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/InferenceFieldMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/InferenceFieldMetadata.java index 271c60e829a87..8917d5a9cbbb5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/InferenceFieldMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/InferenceFieldMetadata.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.TransportVersions; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.SimpleDiffable; import org.elasticsearch.common.io.stream.StreamInput; @@ -23,8 +24,6 @@ import java.util.List; import java.util.Objects; -import static org.elasticsearch.TransportVersions.SEMANTIC_TEXT_SEARCH_INFERENCE_ID; - /** * Contains inference field data for fields. * As inference is done in the coordinator node to avoid re-doing it at shard / replica level, the coordinator needs to check for the need @@ -56,7 +55,7 @@ public InferenceFieldMetadata(String name, String inferenceId, String searchInfe public InferenceFieldMetadata(StreamInput input) throws IOException { this.name = input.readString(); this.inferenceId = input.readString(); - if (input.getTransportVersion().onOrAfter(SEMANTIC_TEXT_SEARCH_INFERENCE_ID)) { + if (input.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.searchInferenceId = input.readString(); } else { this.searchInferenceId = this.inferenceId; @@ -68,7 +67,7 @@ public InferenceFieldMetadata(StreamInput input) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeString(name); out.writeString(inferenceId); - if (out.getTransportVersion().onOrAfter(SEMANTIC_TEXT_SEARCH_INFERENCE_ID)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeString(searchInferenceId); } out.writeStringArray(sourceFields); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index 790b8e4ab75fa..60cf6b10417fa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -317,7 +317,7 @@ public static Diff readDiffFrom(StreamInput in) throws IOException public static RoutingTable readFrom(StreamInput in) throws IOException { Builder builder = new Builder(); - if (in.getTransportVersion().before(TransportVersions.ROUTING_TABLE_VERSION_REMOVED)) { + if (in.getTransportVersion().before(TransportVersions.V_8_16_0)) { in.readLong(); // previously 'version', unused in all applicable versions so any number will do } int size = in.readVInt(); @@ -331,7 +331,7 @@ public static RoutingTable readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().before(TransportVersions.ROUTING_TABLE_VERSION_REMOVED)) { + if (out.getTransportVersion().before(TransportVersions.V_8_16_0)) { out.writeLong(0); // previously 'version', unused in all applicable versions so any number will do } out.writeCollection(indicesRouting.values()); @@ -349,7 +349,7 @@ private static class RoutingTableDiff implements Diff { new DiffableUtils.DiffableValueReader<>(IndexRoutingTable::readFrom, IndexRoutingTable::readDiffFrom); RoutingTableDiff(StreamInput in) throws IOException { - if (in.getTransportVersion().before(TransportVersions.ROUTING_TABLE_VERSION_REMOVED)) { + if (in.getTransportVersion().before(TransportVersions.V_8_16_0)) { in.readLong(); // previously 'version', unused in all applicable versions so any number will do } indicesRouting = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), DIFF_VALUE_READER); @@ -366,7 +366,7 @@ public RoutingTable apply(RoutingTable part) { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().before(TransportVersions.ROUTING_TABLE_VERSION_REMOVED)) { + if (out.getTransportVersion().before(TransportVersions.V_8_16_0)) { out.writeLong(0); // previously 'version', unused in all applicable versions so any number will do } indicesRouting.writeTo(out); diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 644cc6bb69927..e07861ba05433 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -908,11 +908,8 @@ public final Instant readOptionalInstant() throws IOException { private ZonedDateTime readZonedDateTime() throws IOException { final String timeZoneId = readString(); final Instant instant; - if (getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT_BROKEN)) { - // epoch seconds can be negative, but it was incorrectly first written as vlong - boolean zlong = getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT); - long seconds = zlong ? readZLong() : readVLong(); - instant = Instant.ofEpochSecond(seconds, readInt()); + if (getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + instant = Instant.ofEpochSecond(readZLong(), readInt()); } else { instant = Instant.ofEpochMilli(readLong()); } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index d724e5ea25ca6..6738af32f04d6 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -768,13 +768,8 @@ public final void writeOptionalInstant(@Nullable Instant instant) throws IOExcep final ZonedDateTime zonedDateTime = (ZonedDateTime) v; o.writeString(zonedDateTime.getZone().getId()); Instant instant = zonedDateTime.toInstant(); - if (o.getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT_BROKEN)) { - // epoch seconds can be negative, but it was incorrectly first written as vlong - if (o.getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT)) { - o.writeZLong(instant.getEpochSecond()); - } else { - o.writeVLong(instant.getEpochSecond()); - } + if (o.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + o.writeZLong(instant.getEpochSecond()); o.writeInt(instant.getNano()); } else { o.writeLong(instant.toEpochMilli()); diff --git a/server/src/main/java/org/elasticsearch/index/engine/CommitStats.java b/server/src/main/java/org/elasticsearch/index/engine/CommitStats.java index a871524b45e9e..520174a4b3638 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/CommitStats.java +++ b/server/src/main/java/org/elasticsearch/index/engine/CommitStats.java @@ -46,7 +46,7 @@ public CommitStats(SegmentInfos segmentInfos) { generation = in.readLong(); id = in.readOptionalString(); numDocs = in.readInt(); - numLeaves = in.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_LEVEL_FIELDS_STATS) ? in.readVInt() : 0; + numLeaves = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readVInt() : 0; } @Override @@ -100,7 +100,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(generation); out.writeOptionalString(id); out.writeInt(numDocs); - if (out.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_LEVEL_FIELDS_STATS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVInt(numLeaves); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java b/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java index 56210a292995c..10b0856540399 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java @@ -52,7 +52,7 @@ public NodeMappingStats() { public NodeMappingStats(StreamInput in) throws IOException { totalCount = in.readVLong(); totalEstimatedOverhead = in.readVLong(); - if (in.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_LEVEL_FIELDS_STATS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { totalSegments = in.readVLong(); totalSegmentFields = in.readVLong(); } @@ -93,7 +93,7 @@ public long getTotalSegmentFields() { public void writeTo(StreamOutput out) throws IOException { out.writeVLong(totalCount); out.writeVLong(totalEstimatedOverhead); - if (out.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_LEVEL_FIELDS_STATS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVLong(totalSegments); out.writeVLong(totalSegmentFields); } diff --git a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java index 647e45d1beda1..6ae0c4872cfa5 100644 --- a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java +++ b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java @@ -825,7 +825,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.REGEX_AND_RANGE_INTERVAL_QUERIES; + return TransportVersions.V_8_16_0; } @Override @@ -1129,7 +1129,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.REGEX_AND_RANGE_INTERVAL_QUERIES; + return TransportVersions.V_8_16_0; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/RankDocsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RankDocsQueryBuilder.java index 33077697a2ce6..889fa40b79aa1 100644 --- a/server/src/main/java/org/elasticsearch/index/query/RankDocsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/RankDocsQueryBuilder.java @@ -25,8 +25,6 @@ import java.util.Map; import java.util.Objects; -import static org.elasticsearch.TransportVersions.RRF_QUERY_REWRITE; - public class RankDocsQueryBuilder extends AbstractQueryBuilder { public static final String NAME = "rank_docs_query"; @@ -44,7 +42,7 @@ public RankDocsQueryBuilder(RankDoc[] rankDocs, QueryBuilder[] queryBuilders, bo public RankDocsQueryBuilder(StreamInput in) throws IOException { super(in); this.rankDocs = in.readArray(c -> c.readNamedWriteable(RankDoc.class), RankDoc[]::new); - if (in.getTransportVersion().onOrAfter(RRF_QUERY_REWRITE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.queryBuilders = in.readOptionalArray(c -> c.readNamedWriteable(QueryBuilder.class), QueryBuilder[]::new); this.onlyRankDocs = in.readBoolean(); } else { @@ -85,7 +83,7 @@ public RankDoc[] rankDocs() { @Override protected void doWriteTo(StreamOutput out) throws IOException { out.writeArray(StreamOutput::writeNamedWriteable, rankDocs); - if (out.getTransportVersion().onOrAfter(RRF_QUERY_REWRITE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalArray(StreamOutput::writeNamedWriteable, queryBuilders); out.writeBoolean(onlyRankDocs); } @@ -145,6 +143,6 @@ protected int doHashCode() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.RANK_DOCS_RETRIEVER; + return TransportVersions.V_8_16_0; } } diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java b/server/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java index ff514091979c3..8b19d72ccc09d 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java @@ -105,7 +105,7 @@ private Stats(StreamInput in) throws IOException { suggestTimeInMillis = in.readVLong(); suggestCurrent = in.readVLong(); - if (in.getTransportVersion().onOrAfter(TransportVersions.SEARCH_FAILURE_STATS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { queryFailure = in.readVLong(); fetchFailure = in.readVLong(); } @@ -129,7 +129,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(suggestTimeInMillis); out.writeVLong(suggestCurrent); - if (out.getTransportVersion().onOrAfter(TransportVersions.SEARCH_FAILURE_STATS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVLong(queryFailure); out.writeVLong(fetchFailure); } diff --git a/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java b/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java index 9c666bd4a35f5..ee38273f13daf 100644 --- a/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java +++ b/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java @@ -44,7 +44,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_EIS_INTEGRATION_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/server/src/main/java/org/elasticsearch/inference/ModelConfigurations.java b/server/src/main/java/org/elasticsearch/inference/ModelConfigurations.java index ebf32f0411555..53ce0bab63612 100644 --- a/server/src/main/java/org/elasticsearch/inference/ModelConfigurations.java +++ b/server/src/main/java/org/elasticsearch/inference/ModelConfigurations.java @@ -121,7 +121,7 @@ public ModelConfigurations(StreamInput in) throws IOException { this.service = in.readString(); this.serviceSettings = in.readNamedWriteable(ServiceSettings.class); this.taskSettings = in.readNamedWriteable(TaskSettings.class); - this.chunkingSettings = in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_CHUNKING_SETTINGS) + this.chunkingSettings = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalNamedWriteable(ChunkingSettings.class) : null; } @@ -133,7 +133,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(service); out.writeNamedWriteable(serviceSettings); out.writeNamedWriteable(taskSettings); - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_CHUNKING_SETTINGS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalNamedWriteable(chunkingSettings); } } diff --git a/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java b/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java index e696c38b9f017..ff6a687da9b4d 100644 --- a/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java +++ b/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java @@ -64,7 +64,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER; + return TransportVersions.V_8_16_0; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index a1e8eb25f4780..f8d161ef1f5e5 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -263,7 +263,7 @@ private DateTime(DateFormatter formatter, ZoneId timeZone, DateFieldMapper.Resol private DateTime(StreamInput in) throws IOException { String formatterPattern = in.readString(); - Locale locale = in.getTransportVersion().onOrAfter(TransportVersions.DATE_TIME_DOC_VALUES_LOCALES) + Locale locale = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? LocaleUtils.parse(in.readString()) : DateFieldMapper.DEFAULT_LOCALE; String zoneId = in.readString(); @@ -297,7 +297,7 @@ public static DateTime readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(formatter.pattern()); - if (out.getTransportVersion().onOrAfter(TransportVersions.DATE_TIME_DOC_VALUES_LOCALES)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeString(formatter.locale().toString()); } out.writeString(timeZone.getId()); diff --git a/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java b/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java index 9ab14aa9362b5..d4127836a4e4a 100644 --- a/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java +++ b/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java @@ -44,7 +44,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.RANK_DOCS_RETRIEVER; + return TransportVersions.V_8_16_0; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java index c8670a8dfeec2..77d708432cf26 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java @@ -55,8 +55,7 @@ public ExactKnnQueryBuilder(StreamInput in) throws IOException { this.query = VectorData.fromFloats(in.readFloatArray()); } this.field = in.readString(); - if (in.getTransportVersion().onOrAfter(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS) - || in.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_0)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { this.vectorSimilarity = in.readOptionalFloat(); } else { this.vectorSimilarity = null; @@ -88,8 +87,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeFloatArray(query.asFloatVector()); } out.writeString(field); - if (out.getTransportVersion().onOrAfter(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS) - || out.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_0)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeOptionalFloat(vectorSimilarity); } } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java index f52addefc8b1c..b5ba97906f0ec 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java @@ -71,8 +71,7 @@ public KnnScoreDocQueryBuilder(StreamInput in) throws IOException { this.fieldName = null; this.queryVector = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS) - || in.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_0)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { this.vectorSimilarity = in.readOptionalFloat(); } else { this.vectorSimilarity = null; @@ -116,8 +115,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeBoolean(false); } } - if (out.getTransportVersion().onOrAfter(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS) - || out.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_0)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeOptionalFloat(vectorSimilarity); } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/RegisteredPolicySnapshots.java b/server/src/main/java/org/elasticsearch/snapshots/RegisteredPolicySnapshots.java index f34b876697473..231894875b7fa 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RegisteredPolicySnapshots.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RegisteredPolicySnapshots.java @@ -101,7 +101,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.REGISTER_SLM_STATS; + return TransportVersions.V_8_16_0; } @Override @@ -171,7 +171,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.REGISTER_SLM_STATS; + return TransportVersions.V_8_16_0; } } diff --git a/server/src/test/java/org/elasticsearch/TransportVersionTests.java b/server/src/test/java/org/elasticsearch/TransportVersionTests.java index 6c2cc5c1f4cc0..08b12cec2e17e 100644 --- a/server/src/test/java/org/elasticsearch/TransportVersionTests.java +++ b/server/src/test/java/org/elasticsearch/TransportVersionTests.java @@ -211,7 +211,7 @@ public void testDenseTransportVersions() { Set missingVersions = new TreeSet<>(); TransportVersion previous = null; for (var tv : TransportVersions.getAllVersions()) { - if (tv.before(TransportVersions.V_8_15_2)) { + if (tv.before(TransportVersions.V_8_16_0)) { continue; } if (previous == null) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParametersTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParametersTests.java index f37b1d1b41712..cfdbfdfbfcf8c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParametersTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParametersTests.java @@ -23,7 +23,7 @@ public class NodesStatsRequestParametersTests extends ESTestCase { public void testReadWriteMetricSet() { - for (var version : List.of(TransportVersions.V_8_15_0, TransportVersions.NODES_STATS_ENUM_SET)) { + for (var version : List.of(TransportVersions.V_8_15_0, TransportVersions.V_8_16_0)) { var randSet = randomSubsetOf(Metric.ALL); var metricsOut = randSet.isEmpty() ? EnumSet.noneOf(Metric.class) : EnumSet.copyOf(randSet); try { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java index 89ccd4ab63d7f..46b757407e6a9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java @@ -199,7 +199,7 @@ public void testSerializationBWC() throws IOException { randomQueryUsage(QUERY_TYPES.size()), version.onOrAfter(TransportVersions.V_8_12_0) ? randomRescorerUsage(RESCORER_TYPES.size()) : Map.of(), randomSectionsUsage(SECTIONS.size()), - version.onOrAfter(TransportVersions.RETRIEVERS_TELEMETRY_ADDED) ? randomRetrieversUsage(RETRIEVERS.size()) : Map.of(), + version.onOrAfter(TransportVersions.V_8_16_0) ? randomRetrieversUsage(RETRIEVERS.size()) : Map.of(), randomLongBetween(0, Long.MAX_VALUE) ); assertSerialization(testInstance, version); diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java index d2b6d0a6ec6d7..afaa7a9a32888 100644 --- a/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java +++ b/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java @@ -11,6 +11,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -53,8 +54,6 @@ import static java.time.Instant.ofEpochSecond; import static java.time.ZonedDateTime.ofInstant; -import static org.elasticsearch.TransportVersions.ZDT_NANOS_SUPPORT; -import static org.elasticsearch.TransportVersions.ZDT_NANOS_SUPPORT_BROKEN; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; @@ -729,15 +728,11 @@ public void testReadAfterReachingEndOfStream() throws IOException { } public void testZonedDateTimeSerialization() throws IOException { - checkZonedDateTimeSerialization(ZDT_NANOS_SUPPORT); - } - - public void testZonedDateTimeMillisBwcSerializationV1() throws IOException { - checkZonedDateTimeSerialization(TransportVersionUtils.getPreviousVersion(ZDT_NANOS_SUPPORT_BROKEN)); + checkZonedDateTimeSerialization(TransportVersions.V_8_16_0); } public void testZonedDateTimeMillisBwcSerialization() throws IOException { - checkZonedDateTimeSerialization(TransportVersionUtils.getPreviousVersion(ZDT_NANOS_SUPPORT)); + checkZonedDateTimeSerialization(TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_16_0)); } public void checkZonedDateTimeSerialization(TransportVersion tv) throws IOException { @@ -745,12 +740,12 @@ public void checkZonedDateTimeSerialization(TransportVersion tv) throws IOExcept assertGenericRoundtrip(ofInstant(ofEpochSecond(1), randomZone()), tv); // just want to test a large number that will use 5+ bytes long maxEpochSecond = Integer.MAX_VALUE; - long minEpochSecond = tv.between(ZDT_NANOS_SUPPORT_BROKEN, ZDT_NANOS_SUPPORT) ? 0 : Integer.MIN_VALUE; + long minEpochSecond = Integer.MIN_VALUE; assertGenericRoundtrip(ofInstant(ofEpochSecond(maxEpochSecond), randomZone()), tv); assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond)), randomZone()), tv); assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond), 1_000_000), randomZone()), tv); assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond), 999_000_000), randomZone()), tv); - if (tv.onOrAfter(ZDT_NANOS_SUPPORT)) { + if (tv.onOrAfter(TransportVersions.V_8_16_0)) { assertGenericRoundtrip( ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond), 999_999_999), randomZone()), tv diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsage.java index 7a31888a440c3..a61a86eea7104 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsage.java @@ -111,7 +111,7 @@ public LifecycleStats( } public static LifecycleStats read(StreamInput in) throws IOException { - if (in.getTransportVersion().onOrAfter(TransportVersions.GLOBAL_RETENTION_TELEMETRY)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { return new LifecycleStats( in.readVLong(), in.readBoolean(), @@ -139,7 +139,7 @@ public static LifecycleStats read(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.GLOBAL_RETENTION_TELEMETRY)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVLong(dataStreamsWithLifecyclesCount); out.writeBoolean(defaultRolloverUsed); dataRetentionStats.writeTo(out); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java index 0457de6edcc9f..36322ed6c6cbd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java @@ -209,7 +209,7 @@ public CacheStats(StreamInput in) throws IOException { in.readVLong(), in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? in.readLong() : -1, in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? in.readLong() : -1, - in.getTransportVersion().onOrAfter(TransportVersions.ENRICH_CACHE_STATS_SIZE_ADDED) ? in.readLong() : -1 + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readLong() : -1 ); } @@ -237,7 +237,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(hitsTimeInMillis); out.writeLong(missesTimeInMillis); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ENRICH_CACHE_STATS_SIZE_ADDED)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeLong(cacheSizeInBytes); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java index 33402671a2236..5d635c97d9c8c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java @@ -328,7 +328,7 @@ public IndexLifecycleExplainResponse(StreamInput in) throws IOException { } else { indexCreationDate = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.RETAIN_ILM_STEP_INFO)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { previousStepInfo = in.readOptionalBytesReference(); } else { previousStepInfo = null; @@ -379,7 +379,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_1_0)) { out.writeOptionalLong(indexCreationDate); } - if (out.getTransportVersion().onOrAfter(TransportVersions.RETAIN_ILM_STEP_INFO)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalBytesReference(previousStepInfo); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index c06dcc0f083d1..da64df2672bdb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.TransportVersions; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexAbstraction; @@ -32,7 +33,6 @@ import java.util.List; import java.util.Objects; -import static org.elasticsearch.TransportVersions.ILM_ADD_SEARCHABLE_SNAPSHOT_TOTAL_SHARDS_PER_NODE; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_SNAPSHOT_NAME_SETTING_KEY; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY; @@ -102,9 +102,7 @@ public SearchableSnapshotAction(String snapshotRepository) { public SearchableSnapshotAction(StreamInput in) throws IOException { this.snapshotRepository = in.readString(); this.forceMergeIndex = in.readBoolean(); - this.totalShardsPerNode = in.getTransportVersion().onOrAfter(ILM_ADD_SEARCHABLE_SNAPSHOT_TOTAL_SHARDS_PER_NODE) - ? in.readOptionalInt() - : null; + this.totalShardsPerNode = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalInt() : null; } boolean isForceMergeIndex() { @@ -424,7 +422,7 @@ public String getWriteableName() { public void writeTo(StreamOutput out) throws IOException { out.writeString(snapshotRepository); out.writeBoolean(forceMergeIndex); - if (out.getTransportVersion().onOrAfter(ILM_ADD_SEARCHABLE_SNAPSHOT_TOTAL_SHARDS_PER_NODE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalInt(totalShardsPerNode); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java index 226fe3630b387..c3f991a8b4e1e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java @@ -127,7 +127,7 @@ public Response(StreamInput in) throws IOException { pipelineIds = Set.of(); } - if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_DONT_DELETE_WHEN_SEMANTIC_TEXT_EXISTS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { indexes = in.readCollectionAsSet(StreamInput::readString); dryRunMessage = in.readOptionalString(); } else { @@ -143,7 +143,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeCollection(pipelineIds, StreamOutput::writeString); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_DONT_DELETE_WHEN_SEMANTIC_TEXT_EXISTS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeCollection(indexes, StreamOutput::writeString); out.writeOptionalString(dryRunMessage); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceModelAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceModelAction.java index ea0462d0f103e..ba3d417d02672 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceModelAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceModelAction.java @@ -63,7 +63,7 @@ public Request(StreamInput in) throws IOException { this.inferenceEntityId = in.readString(); this.taskType = TaskType.fromStream(in); if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_DONT_PERSIST_ON_READ) - || in.getTransportVersion().isPatchFrom(TransportVersions.INFERENCE_DONT_PERSIST_ON_READ_BACKPORT_8_16)) { + || in.getTransportVersion().isPatchFrom(TransportVersions.V_8_16_0)) { this.persistDefaultConfig = in.readBoolean(); } else { this.persistDefaultConfig = PERSIST_DEFAULT_CONFIGS; @@ -89,7 +89,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(inferenceEntityId); taskType.writeTo(out); if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_DONT_PERSIST_ON_READ) - || out.getTransportVersion().isPatchFrom(TransportVersions.INFERENCE_DONT_PERSIST_ON_READ_BACKPORT_8_16)) { + || out.getTransportVersion().isPatchFrom(TransportVersions.V_8_16_0)) { out.writeBoolean(this.persistDefaultConfig); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsage.java index 0645299dfc30e..8c4611f05e72a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsage.java @@ -66,7 +66,7 @@ public MachineLearningFeatureSetUsage(StreamInput in) throws IOException { this.analyticsUsage = in.readGenericMap(); this.inferenceUsage = in.readGenericMap(); this.nodeCount = in.readInt(); - if (in.getTransportVersion().onOrAfter(TransportVersions.ML_TELEMETRY_MEMORY_ADDED)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.memoryUsage = in.readGenericMap(); } else { this.memoryUsage = Map.of(); @@ -86,7 +86,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeGenericMap(analyticsUsage); out.writeGenericMap(inferenceUsage); out.writeInt(nodeCount); - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_TELEMETRY_MEMORY_ADDED)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeGenericMap(memoryUsage); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAssignmentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAssignmentAction.java index c6976ab4b513e..2aedb46347534 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAssignmentAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAssignmentAction.java @@ -47,7 +47,7 @@ public Request(StartTrainedModelDeploymentAction.TaskParams taskParams, Adaptive public Request(StreamInput in) throws IOException { super(in); this.taskParams = new StartTrainedModelDeploymentAction.TaskParams(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.adaptiveAllocationsSettings = in.readOptionalWriteable(AdaptiveAllocationsSettings::new); } else { this.adaptiveAllocationsSettings = null; @@ -63,7 +63,7 @@ public ActionRequestValidationException validate() { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); taskParams.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalWriteable(adaptiveAllocationsSettings); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java index b298d486c9e03..1bf92262b30fb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java @@ -169,7 +169,7 @@ public Request(StreamInput in) throws IOException { modelId = in.readString(); timeout = in.readTimeValue(); waitForState = in.readEnum(AllocationStatus.State.class); - if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { numberOfAllocations = in.readOptionalVInt(); } else { numberOfAllocations = in.readVInt(); @@ -189,7 +189,7 @@ public Request(StreamInput in) throws IOException { } else { this.deploymentId = modelId; } - if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.adaptiveAllocationsSettings = in.readOptionalWriteable(AdaptiveAllocationsSettings::new); } else { this.adaptiveAllocationsSettings = null; @@ -297,7 +297,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(modelId); out.writeTimeValue(timeout); out.writeEnum(waitForState); - if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalVInt(numberOfAllocations); } else { out.writeVInt(numberOfAllocations); @@ -313,7 +313,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) { out.writeString(deploymentId); } - if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalWriteable(adaptiveAllocationsSettings); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java index cb578fdb157de..2018c9526ec83 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java @@ -87,7 +87,7 @@ public Request(String deploymentId) { public Request(StreamInput in) throws IOException { super(in); deploymentId = in.readString(); - if (in.getTransportVersion().before(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (in.getTransportVersion().before(TransportVersions.V_8_16_0)) { numberOfAllocations = in.readVInt(); adaptiveAllocationsSettings = null; isInternal = false; @@ -134,7 +134,7 @@ public AdaptiveAllocationsSettings getAdaptiveAllocationsSettings() { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(deploymentId); - if (out.getTransportVersion().before(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (out.getTransportVersion().before(TransportVersions.V_8_16_0)) { out.writeVInt(numberOfAllocations); } else { out.writeOptionalVInt(numberOfAllocations); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java index b007c1da451f5..742daa1bf6137 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java @@ -115,7 +115,7 @@ public ScheduledEvent(StreamInput in) throws IOException { description = in.readString(); startTime = in.readInstant(); endTime = in.readInstant(); - if (in.getTransportVersion().onOrAfter(TransportVersions.ML_SCHEDULED_EVENT_TIME_SHIFT_CONFIGURATION)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { skipResult = in.readBoolean(); skipModelUpdate = in.readBoolean(); forceTimeShift = in.readOptionalInt(); @@ -204,7 +204,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(description); out.writeInstant(startTime); out.writeInstant(endTime); - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_SCHEDULED_EVENT_TIME_SHIFT_CONFIGURATION)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeBoolean(skipResult); out.writeBoolean(skipModelUpdate); out.writeOptionalInt(forceTimeShift); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java index 858d97bf6f956..31b513eea161e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java @@ -483,7 +483,7 @@ public AssignmentStats(StreamInput in) throws IOException { } else { deploymentId = modelId; } - if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { adaptiveAllocationsSettings = in.readOptionalWriteable(AdaptiveAllocationsSettings::new); } else { adaptiveAllocationsSettings = null; @@ -666,7 +666,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) { out.writeString(deploymentId); } - if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalWriteable(adaptiveAllocationsSettings); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java index efd07cceae09b..249e27d6f25e0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java @@ -178,7 +178,7 @@ public TrainedModelAssignment(StreamInput in) throws IOException { } else { this.maxAssignedAllocations = totalCurrentAllocations(); } - if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.adaptiveAllocationsSettings = in.readOptionalWriteable(AdaptiveAllocationsSettings::new); } else { this.adaptiveAllocationsSettings = null; @@ -382,7 +382,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_4_0)) { out.writeVInt(maxAssignedAllocations); } - if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalWriteable(adaptiveAllocationsSettings); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfig.java index 9929e59a9c803..a4d7c9c7fa08f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfig.java @@ -41,7 +41,6 @@ public class LearningToRankConfig extends RegressionConfig implements Rewriteable { public static final ParseField NAME = new ParseField("learning_to_rank"); - static final TransportVersion MIN_SUPPORTED_TRANSPORT_VERSION = TransportVersions.LTR_SERVERLESS_RELEASE; public static final ParseField NUM_TOP_FEATURE_IMPORTANCE_VALUES = new ParseField("num_top_feature_importance_values"); public static final ParseField FEATURE_EXTRACTORS = new ParseField("feature_extractors"); public static final ParseField DEFAULT_PARAMS = new ParseField("default_params"); @@ -226,7 +225,7 @@ public MlConfigVersion getMinimalSupportedMlConfigVersion() { @Override public TransportVersion getMinimalSupportedTransportVersion() { - return MIN_SUPPORTED_TRANSPORT_VERSION; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java index eb952a7dc7e5c..4bdced325311f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java @@ -68,7 +68,7 @@ public DetectionRule(StreamInput in) throws IOException { actions = in.readEnumSet(RuleAction.class); scope = new RuleScope(in); conditions = in.readCollectionAsList(RuleCondition::new); - if (in.getTransportVersion().onOrAfter(TransportVersions.ML_ADD_DETECTION_RULE_PARAMS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { params = new RuleParams(in); } else { params = new RuleParams(); @@ -80,7 +80,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeEnumSet(actions); scope.writeTo(out); out.writeCollection(conditions); - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_ADD_DETECTION_RULE_PARAMS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { params.writeTo(out); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivileges.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivileges.java index b93aa079a28d2..148fdf21fd2df 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivileges.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivileges.java @@ -82,7 +82,7 @@ public static ConfigurableClusterPrivilege[] readArray(StreamInput in) throws IO * Utility method to write an array of {@link ConfigurableClusterPrivilege} objects to a {@link StreamOutput} */ public static void writeArray(StreamOutput out, ConfigurableClusterPrivilege[] privileges) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeArray(WRITER, privileges); } else { out.writeArray( diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsageTests.java index 87d658c6f983c..e9ec8dfe8ee52 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsageTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsageTests.java @@ -57,7 +57,7 @@ protected MachineLearningFeatureSetUsage mutateInstance(MachineLearningFeatureSe @Override protected MachineLearningFeatureSetUsage mutateInstanceForVersion(MachineLearningFeatureSetUsage instance, TransportVersion version) { - if (version.before(TransportVersions.ML_TELEMETRY_MEMORY_ADDED)) { + if (version.before(TransportVersions.V_8_16_0)) { return new MachineLearningFeatureSetUsage( instance.available(), instance.enabled(), diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java index 3a61c848d3813..d694b2681ee88 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java @@ -68,8 +68,7 @@ public QueryRulesetListItem(StreamInput in) throws IOException { this.criteriaTypeToCountMap = Map.of(); } TransportVersion streamTransportVersion = in.getTransportVersion(); - if (streamTransportVersion.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_15) - || streamTransportVersion.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16) + if (streamTransportVersion.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16) || streamTransportVersion.onOrAfter(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES)) { this.ruleTypeToCountMap = in.readMap(m -> in.readEnum(QueryRule.QueryRuleType.class), StreamInput::readInt); } else { @@ -104,8 +103,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(criteriaTypeToCountMap, StreamOutput::writeEnum, StreamOutput::writeInt); } TransportVersion streamTransportVersion = out.getTransportVersion(); - if (streamTransportVersion.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_15) - || streamTransportVersion.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16) + if (streamTransportVersion.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16) || streamTransportVersion.onOrAfter(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES)) { out.writeMap(ruleTypeToCountMap, StreamOutput::writeEnum, StreamOutput::writeInt); } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java index 27d5e240534b2..c822dd123d3f8 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java @@ -59,8 +59,7 @@ protected ListQueryRulesetsAction.Response mutateInstanceForVersion( ListQueryRulesetsAction.Response instance, TransportVersion version ) { - if (version.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_15) - || version.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16) + if (version.isPatchFrom(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16) || version.onOrAfter(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES)) { return instance; } else if (version.onOrAfter(QueryRulesetListItem.EXPANDED_RULESET_COUNT_TRANSPORT_VERSION)) { diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionRequestBWCSerializingTests.java index 7041de1106b50..8582ee1bd8d24 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionRequestBWCSerializingTests.java @@ -51,6 +51,6 @@ protected TestQueryRulesetAction.Request mutateInstanceForVersion(TestQueryRules @Override protected List bwcVersions() { - return getAllBWCVersions().stream().filter(v -> v.onOrAfter(TransportVersions.QUERY_RULE_TEST_API)).collect(Collectors.toList()); + return getAllBWCVersions().stream().filter(v -> v.onOrAfter(TransportVersions.V_8_16_0)).collect(Collectors.toList()); } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionResponseBWCSerializingTests.java index a6562fb7b52af..142310ac40332 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/TestQueryRulesetActionResponseBWCSerializingTests.java @@ -47,6 +47,6 @@ protected TestQueryRulesetAction.Response mutateInstanceForVersion(TestQueryRule @Override protected List bwcVersions() { - return getAllBWCVersions().stream().filter(v -> v.onOrAfter(TransportVersions.QUERY_RULE_TEST_API)).collect(Collectors.toList()); + return getAllBWCVersions().stream().filter(v -> v.onOrAfter(TransportVersions.V_8_16_0)).collect(Collectors.toList()); } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java index 47dadcbb11de2..73e2d5ec626ac 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java @@ -72,7 +72,7 @@ public EsField(StreamInput in) throws IOException { private DataType readDataType(StreamInput in) throws IOException { String name = readCachedStringWithVersionCheck(in); - if (in.getTransportVersion().before(TransportVersions.ESQL_NESTED_UNSUPPORTED) && name.equalsIgnoreCase("NESTED")) { + if (in.getTransportVersion().before(TransportVersions.V_8_16_0) && name.equalsIgnoreCase("NESTED")) { /* * The "nested" data type existed in older versions of ESQL but was * entirely used to filter mappings away. Those versions will still diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java index e8ccae3429001..b570a50535a59 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java @@ -52,7 +52,7 @@ public interface PlanStreamInput { String readCachedString() throws IOException; static String readCachedStringWithVersionCheck(StreamInput planStreamInput) throws IOException { - if (planStreamInput.getTransportVersion().before(TransportVersions.ESQL_CACHED_STRING_SERIALIZATION)) { + if (planStreamInput.getTransportVersion().before(TransportVersions.V_8_16_0)) { return planStreamInput.readString(); } return ((PlanStreamInput) planStreamInput).readCachedString(); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java index fb4af33d2fd60..a5afcb5fa29a6 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java @@ -37,7 +37,7 @@ public interface PlanStreamOutput { void writeCachedString(String field) throws IOException; static void writeCachedStringWithVersionCheck(StreamOutput planStreamOutput, String string) throws IOException { - if (planStreamOutput.getTransportVersion().before(TransportVersions.ESQL_CACHED_STRING_SERIALIZATION)) { + if (planStreamOutput.getTransportVersion().before(TransportVersions.V_8_16_0)) { planStreamOutput.writeString(string); } else { ((PlanStreamOutput) planStreamOutput).writeCachedString(string); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java index 9338077a55570..f57f450c7ee39 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java @@ -219,7 +219,7 @@ public Status(long aggregationNanos, long aggregationFinishNanos, int pagesProce protected Status(StreamInput in) throws IOException { aggregationNanos = in.readVLong(); - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_AGGREGATION_OPERATOR_STATUS_FINISH_NANOS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { aggregationFinishNanos = in.readOptionalVLong(); } else { aggregationFinishNanos = null; @@ -230,7 +230,7 @@ protected Status(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(aggregationNanos); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_AGGREGATION_OPERATOR_STATUS_FINISH_NANOS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalVLong(aggregationFinishNanos); } out.writeVInt(pagesProcessed); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java index d98613f1817ab..c071b5055df76 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java @@ -79,7 +79,7 @@ public DriverProfile( } public DriverProfile(StreamInput in) throws IOException { - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PROFILE_SLEEPS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.startMillis = in.readVLong(); this.stopMillis = in.readVLong(); } else { @@ -101,7 +101,7 @@ public DriverProfile(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_PROFILE_SLEEPS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVLong(startMillis); out.writeVLong(stopMillis); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java index 01e9a73c4fb5f..d8856ebedb80b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java @@ -76,7 +76,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws static final int RECORDS = 10; public static DriverSleeps read(StreamInput in) throws IOException { - if (in.getTransportVersion().before(TransportVersions.ESQL_PROFILE_SLEEPS)) { + if (in.getTransportVersion().before(TransportVersions.V_8_16_0)) { return empty(); } return new DriverSleeps( @@ -88,7 +88,7 @@ public static DriverSleeps read(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().before(TransportVersions.ESQL_PROFILE_SLEEPS)) { + if (out.getTransportVersion().before(TransportVersions.V_8_16_0)) { return; } out.writeMap(counts, StreamOutput::writeVLong); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java index ba7a7e8266845..52170dfb05256 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java @@ -107,7 +107,7 @@ public EsqlExecutionInfo(StreamInput in) throws IOException { clusterList.forEach(c -> m.put(c.getClusterAlias(), c)); this.clusterInfo = m; } - if (in.getTransportVersion().onOrAfter(TransportVersions.OPT_IN_ESQL_CCS_EXECUTION_INFO)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.includeCCSMetadata = in.readBoolean(); } else { this.includeCCSMetadata = false; @@ -124,7 +124,7 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeCollection(Collections.emptyList()); } - if (out.getTransportVersion().onOrAfter(TransportVersions.OPT_IN_ESQL_CCS_EXECUTION_INFO)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeBoolean(includeCCSMetadata); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java index 77aed298baea5..dc0e9fd1fb06d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java @@ -113,7 +113,7 @@ static EsqlQueryResponse deserialize(BlockStreamInput in) throws IOException { } boolean columnar = in.readBoolean(); EsqlExecutionInfo executionInfo = null; - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { executionInfo = in.readOptionalWriteable(EsqlExecutionInfo::new); } return new EsqlQueryResponse(columns, pages, profile, columnar, asyncExecutionId, isRunning, isAsync, executionInfo); @@ -132,7 +132,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(profile); } out.writeBoolean(columnar); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalWriteable(executionInfo); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java index f7e6793fc4fb3..f7fd991a9ef16 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java @@ -58,7 +58,7 @@ void executeRemoteRequest( ActionListener remoteListener ) { remoteClient.getConnection(remoteRequest, remoteListener.delegateFailure((l, conn) -> { - var remoteAction = conn.getTransportVersion().onOrAfter(TransportVersions.ESQL_ORIGINAL_INDICES) + var remoteAction = conn.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? RESOLVE_REMOTE_TYPE : TransportFieldCapabilitiesAction.REMOTE_TYPE; remoteClient.execute(conn, remoteAction, remoteRequest, l); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ResolvedEnrichPolicy.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ResolvedEnrichPolicy.java index e891089aa55b5..64595e776a96e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ResolvedEnrichPolicy.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ResolvedEnrichPolicy.java @@ -35,8 +35,7 @@ public ResolvedEnrichPolicy(StreamInput in) throws IOException { } private static Reader getEsFieldReader(StreamInput in) { - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION) - || in.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_2)) { return EsField::readFrom; } return EsField::new; @@ -56,8 +55,7 @@ public void writeTo(StreamOutput out) throws IOException { */ (o, v) -> { var field = new EsField(v.getName(), v.getDataType(), v.getProperties(), v.isAggregatable(), v.isAlias()); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION) - || out.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_2)) { field.writeTo(o); } else { field.writeContent(o); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java index d372eddb961ae..089f6db373c54 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java @@ -81,8 +81,7 @@ private UnsupportedAttribute(StreamInput in) throws IOException { this( Source.readFrom((PlanStreamInput) in), readCachedStringWithVersionCheck(in), - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION) - || in.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2) ? EsField.readFrom(in) : new UnsupportedEsField(in), + in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_2) ? EsField.readFrom(in) : new UnsupportedEsField(in), in.readOptionalString(), NameId.readFrom((PlanStreamInput) in) ); @@ -93,8 +92,7 @@ public void writeTo(StreamOutput out) throws IOException { if (((PlanStreamOutput) out).writeAttributeCacheHeader(this)) { Source.EMPTY.writeTo(out); writeCachedStringWithVersionCheck(out, name()); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION) - || out.getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_2)) { field().writeTo(out); } else { field().writeContent(out); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java index 87efccfc90ab3..265b08de5556d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java @@ -53,10 +53,8 @@ protected AggregateFunction(StreamInput in) throws IOException { this( Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) - ? in.readNamedWriteable(Expression.class) - : Literal.TRUE, - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteable(Expression.class) : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteableCollectionAsList(Expression.class) : emptyList() ); @@ -66,7 +64,7 @@ protected AggregateFunction(StreamInput in) throws IOException { public final void writeTo(StreamOutput out) throws IOException { Source.EMPTY.writeTo(out); out.writeNamedWriteable(field); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeNamedWriteable(filter); out.writeNamedWriteableCollection(parameters); } else { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java index 2e45b1c1fe082..7436db9e00dd2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java @@ -147,10 +147,8 @@ private CountDistinct(StreamInput in) throws IOException { this( Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) - ? in.readNamedWriteable(Expression.class) - : Literal.TRUE, - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteable(Expression.class) : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteableCollectionAsList(Expression.class) : nullSafeList(in.readOptionalNamedWriteable(Expression.class)) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FromPartial.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FromPartial.java index 0f9037a28d7d7..a67b87c7617c4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FromPartial.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FromPartial.java @@ -58,10 +58,8 @@ private FromPartial(StreamInput in) throws IOException { this( Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) - ? in.readNamedWriteable(Expression.class) - : Literal.TRUE, - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteable(Expression.class) : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteableCollectionAsList(Expression.class).get(0) : in.readNamedWriteable(Expression.class) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java index febd9f28b2291..0d57267da1e29 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java @@ -92,10 +92,8 @@ private Percentile(StreamInput in) throws IOException { this( Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) - ? in.readNamedWriteable(Expression.class) - : Literal.TRUE, - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteable(Expression.class) : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteableCollectionAsList(Expression.class).get(0) : in.readNamedWriteable(Expression.class) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java index b7b04658f8d58..87ac9b77a6826 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java @@ -74,10 +74,8 @@ public Rate(StreamInput in) throws IOException { this( Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) - ? in.readNamedWriteable(Expression.class) - : Literal.TRUE, - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteable(Expression.class) : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteableCollectionAsList(Expression.class) : nullSafeList(in.readNamedWriteable(Expression.class), in.readOptionalNamedWriteable(Expression.class)) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ToPartial.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ToPartial.java index cffac616b3c8c..a2856f60e4c51 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ToPartial.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ToPartial.java @@ -80,10 +80,8 @@ private ToPartial(StreamInput in) throws IOException { this( Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) - ? in.readNamedWriteable(Expression.class) - : Literal.TRUE, - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteable(Expression.class) : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteableCollectionAsList(Expression.class).get(0) : in.readNamedWriteable(Expression.class) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java index e0a7da806b3ac..40777b4d78dc2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java @@ -81,10 +81,8 @@ private Top(StreamInput in) throws IOException { super( Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) - ? in.readNamedWriteable(Expression.class) - : Literal.TRUE, - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteable(Expression.class) : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteableCollectionAsList(Expression.class) : asList(in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java index dbcc50cea3b9b..49c68d002440f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java @@ -68,10 +68,8 @@ private WeightedAvg(StreamInput in) throws IOException { this( Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) - ? in.readNamedWriteable(Expression.class) - : Literal.TRUE, - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PER_AGGREGATE_FILTER) + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteable(Expression.class) : Literal.TRUE, + in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readNamedWriteableCollectionAsList(Expression.class).get(0) : in.readNamedWriteable(Expression.class) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java index ce52b3a7611b3..ee51a6f391a65 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java @@ -50,7 +50,7 @@ public void writeTo(StreamOutput out) throws IOException { @SuppressWarnings("unchecked") private static Map readIndexNameWithModes(StreamInput in) throws IOException { - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_INDEX_MODE_CONCRETE_INDICES)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { return in.readMap(IndexMode::readFrom); } else { Set indices = (Set) in.readGenericValue(); @@ -60,7 +60,7 @@ private static Map readIndexNameWithModes(StreamInput in) thr } private static void writeIndexNameWithModes(Map concreteIndices, StreamOutput out) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_INDEX_MODE_CONCRETE_INDICES)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeMap(concreteIndices, (o, v) -> IndexMode.writeTo(v, out)); } else { out.writeGenericValue(concreteIndices.keySet()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java index 47e5b9acfbf9d..948fd1c683544 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java @@ -182,8 +182,7 @@ public NameId mapNameId(long l) { @Override @SuppressWarnings("unchecked") public A readAttributeWithCache(CheckedFunction constructor) throws IOException { - if (getTransportVersion().onOrAfter(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION) - || getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) { + if (getTransportVersion().onOrAfter(TransportVersions.V_8_15_2)) { // it's safe to cast to int, since the max value for this is {@link PlanStreamOutput#MAX_SERIALIZED_ATTRIBUTES} int cacheId = Math.toIntExact(readZLong()); if (cacheId < 0) { @@ -222,8 +221,7 @@ private void cacheAttribute(int id, Attribute attr) { @SuppressWarnings("unchecked") public A readEsFieldWithCache() throws IOException { - if (getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION) - || getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) { + if (getTransportVersion().onOrAfter(TransportVersions.V_8_15_2)) { // it's safe to cast to int, since the max value for this is {@link PlanStreamOutput#MAX_SERIALIZED_ATTRIBUTES} int cacheId = Math.toIntExact(readZLong()); if (cacheId < 0) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java index 615c4266620c7..63d95c21d7d9d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java @@ -154,8 +154,7 @@ public void writeCachedBlock(Block block) throws IOException { @Override public boolean writeAttributeCacheHeader(Attribute attribute) throws IOException { - if (getTransportVersion().onOrAfter(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION) - || getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) { + if (getTransportVersion().onOrAfter(TransportVersions.V_8_15_2)) { Integer cacheId = attributeIdFromCache(attribute); if (cacheId != null) { writeZLong(cacheId); @@ -186,8 +185,7 @@ private int cacheAttribute(Attribute attr) { @Override public boolean writeEsFieldCacheHeader(EsField field) throws IOException { - if (getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION) - || getTransportVersion().isPatchFrom(TransportVersions.V_8_15_2)) { + if (getTransportVersion().onOrAfter(TransportVersions.V_8_15_2)) { Integer cacheId = esFieldIdFromCache(field); if (cacheId != null) { writeZLong(cacheId); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java index dff55f0738975..891d03c571b27 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java @@ -85,7 +85,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeNamedWriteable(child()); out.writeNamedWriteableCollection(groupings()); out.writeNamedWriteableCollection(aggregates()); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_AGGREGATE_EXEC_TRACKS_INTERMEDIATE_ATTRS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeEnum(getMode()); out.writeNamedWriteableCollection(intermediateAttributes()); } else { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeResponse.java index 308192704fe0e..8d2e092cd4149 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeResponse.java @@ -61,7 +61,7 @@ final class ComputeResponse extends TransportResponse { } else { profiles = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.took = in.readOptionalTimeValue(); this.totalShards = in.readVInt(); this.successfulShards = in.readVInt(); @@ -86,7 +86,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(profiles); } } - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalTimeValue(took); out.writeVInt(totalShards); out.writeVInt(successfulShards); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java index 8f890e63bf54e..4c01d326ed7bc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java @@ -81,7 +81,7 @@ final class DataNodeRequest extends TransportRequest implements IndicesRequest.R this.shardIds = in.readCollectionAsList(ShardId::new); this.aliasFilters = in.readMap(Index::new, AliasFilter::readFrom); this.plan = new PlanStreamInput(in, in.namedWriteableRegistry(), configuration).readNamedWriteable(PhysicalPlan.class); - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ORIGINAL_INDICES)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.indices = in.readStringArray(); this.indicesOptions = IndicesOptions.readIndicesOptions(in); } else { @@ -101,7 +101,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(shardIds); out.writeMap(aliasFilters); new PlanStreamOutput(out, configuration).writeNamedWriteable(plan); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ORIGINAL_INDICES)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeStringArray(indices); indicesOptions.writeIndicesOptions(out); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/RemoteClusterPlan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/RemoteClusterPlan.java index 031bfd7139a84..aed196f963e9b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/RemoteClusterPlan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/RemoteClusterPlan.java @@ -23,7 +23,7 @@ static RemoteClusterPlan from(PlanStreamInput planIn) throws IOException { var plan = planIn.readNamedWriteable(PhysicalPlan.class); var targetIndices = planIn.readStringArray(); final OriginalIndices originalIndices; - if (planIn.getTransportVersion().onOrAfter(TransportVersions.ESQL_ORIGINAL_INDICES)) { + if (planIn.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { originalIndices = OriginalIndices.readOriginalIndices(planIn); } else { // fallback to the previous behavior @@ -35,7 +35,7 @@ static RemoteClusterPlan from(PlanStreamInput planIn) throws IOException { public void writeTo(PlanStreamOutput out) throws IOException { out.writeNamedWriteable(plan); out.writeStringArray(targetIndices); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ORIGINAL_INDICES)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { OriginalIndices.writeOriginalIndices(originalIndices, out); } else { out.writeStringArray(originalIndices.indices()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index 8d33e9b480594..bc11d246904d5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -107,7 +107,7 @@ public static class Builder extends AbstractQueryBuilder { super(in); this.next = in.readNamedWriteable(QueryBuilder.class); this.field = in.readString(); - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_SINGLE_VALUE_QUERY_SOURCE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { if (in instanceof PlanStreamInput psi) { this.source = Source.readFrom(psi); } else { @@ -128,7 +128,7 @@ public static class Builder extends AbstractQueryBuilder { protected void doWriteTo(StreamOutput out) throws IOException { out.writeNamedWriteable(next); out.writeString(field); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_SINGLE_VALUE_QUERY_SOURCE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { source.writeTo(out); } else if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { writeOldSource(out, source); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java index 4ec2746b24ee4..997f3265803f7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java @@ -101,7 +101,7 @@ public Configuration(BlockStreamInput in) throws IOException { } else { this.tables = Map.of(); } - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.queryStartTimeNanos = in.readLong(); } else { this.queryStartTimeNanos = -1; @@ -127,7 +127,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeMap(tables, (o1, columns) -> o1.writeMap(columns, StreamOutput::writeWriteable)); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_EXECUTION_INFO)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeLong(queryStartTimeNanos); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java index 07ca112e8c527..3dfc0f611eb2b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java @@ -156,11 +156,7 @@ protected ClusterComputeRequest mutateInstance(ClusterComputeRequest in) throws public void testFallbackIndicesOptions() throws Exception { ClusterComputeRequest request = createTestInstance(); - var version = TransportVersionUtils.randomVersionBetween( - random(), - TransportVersions.V_8_14_0, - TransportVersions.ESQL_ORIGINAL_INDICES - ); + var version = TransportVersionUtils.randomVersionBetween(random(), TransportVersions.V_8_14_0, TransportVersions.V_8_16_0); ClusterComputeRequest cloned = copyInstance(request, version); assertThat(cloned.clusterAlias(), equalTo(request.clusterAlias())); assertThat(cloned.sessionId(), equalTo(request.sessionId())); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java index def52e97666f9..9d6f5bb89218f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java @@ -49,7 +49,7 @@ public SentenceBoundaryChunkingSettings(Integer maxChunkSize, @Nullable Integer public SentenceBoundaryChunkingSettings(StreamInput in) throws IOException { maxChunkSize = in.readInt(); - if (in.getTransportVersion().onOrAfter(TransportVersions.CHUNK_SENTENCE_OVERLAP_SETTING_ADDED)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { sentenceOverlap = in.readVInt(); } } @@ -113,13 +113,13 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_CHUNKING_SETTINGS; + return TransportVersions.V_8_16_0; } @Override public void writeTo(StreamOutput out) throws IOException { out.writeInt(maxChunkSize); - if (out.getTransportVersion().onOrAfter(TransportVersions.CHUNK_SENTENCE_OVERLAP_SETTING_ADDED)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVInt(sentenceOverlap); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettings.java index 7fb0fdc91bf72..7e0378d5b0cd1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettings.java @@ -104,7 +104,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_CHUNKING_SETTINGS; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankBuilder.java index fdb5503e491eb..15d41301d0a3c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankBuilder.java @@ -85,7 +85,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.RANDOM_RERANKER_RETRIEVER; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java index d208623e53324..7ad3e8eea0538 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java @@ -98,6 +98,6 @@ protected void doToXContent(XContentBuilder builder, Params params) throws IOExc @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.TEXT_SIMILARITY_RERANKER_QUERY_REWRITE; + return TransportVersions.V_8_16_0; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java index d7ac7caed7efc..5adc2a11b19d9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java @@ -359,7 +359,7 @@ public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED; + return TransportVersions.V_8_16_0; } public static class Configuration { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceSettings.java index 3500bdf814e16..f6ddac34a2b27 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceSettings.java @@ -163,7 +163,7 @@ public ToXContentObject getFilteredXContentObject() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettings.java index 631ec8a8648e8..a299cf5b655c5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettings.java @@ -74,7 +74,7 @@ public ToXContentObject getFilteredXContentObject() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettings.java index 05b5873a81d8d..7883e7b1d90df 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettings.java @@ -115,7 +115,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsServiceSettings.java index 8896e983d3e7f..8f40ce2a8b8b7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsServiceSettings.java @@ -135,7 +135,7 @@ public ToXContentObject getFilteredXContentObject() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java index 9a431717d9fb9..a08ca6cce66d6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java @@ -151,7 +151,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankServiceSettings.java index 42c7238aefa7f..40e645074f61c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankServiceSettings.java @@ -74,7 +74,7 @@ public ToXContentObject getFilteredXContentObject() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java index 40c3dee00d6c7..2a7806f4beab3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java @@ -85,7 +85,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseServiceSettings.java index fe44c936c4e61..0a55d2aba6cea 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseServiceSettings.java @@ -74,7 +74,7 @@ public ToXContentObject getFilteredXContentObject() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java index 0f4ebce920167..17c5b178c2a13 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java @@ -164,7 +164,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java index a3d2483a068e2..78178466f9f3a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java @@ -92,7 +92,7 @@ public CohereRerankServiceSettings(@Nullable String url, @Nullable String modelI public CohereRerankServiceSettings(StreamInput in) throws IOException { this.uri = createOptionalUri(in.readOptionalString()); - if (in.getTransportVersion().before(TransportVersions.ML_INFERENCE_COHERE_UNUSED_RERANK_SETTINGS_REMOVED)) { + if (in.getTransportVersion().before(TransportVersions.V_8_16_0)) { // An older node sends these fields, so we need to skip them to progress through the serialized data in.readOptionalEnum(SimilarityMeasure.class); in.readOptionalVInt(); @@ -162,7 +162,7 @@ public void writeTo(StreamOutput out) throws IOException { var uriToWrite = uri != null ? uri.toString() : null; out.writeOptionalString(uriToWrite); - if (out.getTransportVersion().before(TransportVersions.ML_INFERENCE_COHERE_UNUSED_RERANK_SETTINGS_REMOVED)) { + if (out.getTransportVersion().before(TransportVersions.V_8_16_0)) { // An old node expects this data to be present, so we need to send at least the booleans // indicating that the fields are not set out.writeOptionalEnum(null); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java index 1f08c06edaa91..b256861e7dd27 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java @@ -229,7 +229,7 @@ public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, M @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_EIS_INTEGRATION_ADDED; + return TransportVersions.V_8_16_0; } private ElasticInferenceServiceModel createModelFromPersistent( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java index bbda1bb716794..3af404aeef36b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java @@ -113,7 +113,7 @@ public RateLimitSettings rateLimitSettings() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_EIS_INTEGRATION_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java index 962c939146ef2..244108edc3dd4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java @@ -157,19 +157,17 @@ public ElasticsearchInternalServiceSettings(ElasticsearchInternalServiceSettings } public ElasticsearchInternalServiceSettings(StreamInput in) throws IOException { - if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.numAllocations = in.readOptionalVInt(); } else { this.numAllocations = in.readVInt(); } this.numThreads = in.readVInt(); this.modelId = in.readString(); - this.adaptiveAllocationsSettings = in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS) + this.adaptiveAllocationsSettings = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalWriteable(AdaptiveAllocationsSettings::new) : null; - this.deploymentId = in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ATTACH_TO_EXISTSING_DEPLOYMENT) - ? in.readOptionalString() - : null; + this.deploymentId = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalString() : null; } public void setNumAllocations(Integer numAllocations) { @@ -178,17 +176,15 @@ public void setNumAllocations(Integer numAllocations) { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalVInt(getNumAllocations()); } else { out.writeVInt(getNumAllocations()); } out.writeVInt(getNumThreads()); out.writeString(modelId()); - if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalWriteable(getAdaptiveAllocationsSettings()); - } - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ATTACH_TO_EXISTSING_DEPLOYMENT)) { out.writeOptionalString(deploymentId); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java index ea263fb77a2da..981a3e95808ef 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java @@ -223,7 +223,7 @@ public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, M @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_IBM_WATSONX_EMBEDDINGS_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/embeddings/IbmWatsonxEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/embeddings/IbmWatsonxEmbeddingsServiceSettings.java index 53d5c6c8bb5e8..3a9625aef31c7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/embeddings/IbmWatsonxEmbeddingsServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/embeddings/IbmWatsonxEmbeddingsServiceSettings.java @@ -207,7 +207,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_IBM_WATSONX_EMBEDDINGS_ADDED; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilder.java index 46edcf1f63c01..b59ef0c40e4f9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilder.java @@ -304,7 +304,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.LTR_SERVERLESS_RELEASE; + return TransportVersions.V_8_16_0; } @Override diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java index 4cd10801b298c..84961f8442163 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java @@ -62,7 +62,7 @@ public RRFRankDoc(StreamInput in) throws IOException { rank = in.readVInt(); positions = in.readIntArray(); scores = in.readFloatArray(); - if (in.getTransportVersion().onOrAfter(TransportVersions.RRF_QUERY_REWRITE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.rankConstant = in.readVInt(); } else { this.rankConstant = DEFAULT_RANK_CONSTANT; @@ -119,7 +119,7 @@ public void doWriteTo(StreamOutput out) throws IOException { out.writeVInt(rank); out.writeIntArray(positions); out.writeFloatArray(scores); - if (out.getTransportVersion().onOrAfter(TransportVersions.RRF_QUERY_REWRITE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVInt(rankConstant); } } @@ -173,6 +173,6 @@ protected void doToXContent(XContentBuilder builder, Params params) throws IOExc @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.RRF_QUERY_REWRITE; + return TransportVersions.V_8_16_0; } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 03558e72fdca3..c1be25b27c51e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -14,6 +14,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.DocWriteRequest; @@ -138,7 +139,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.elasticsearch.TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.search.SearchService.DEFAULT_KEEPALIVE_SETTING; import static org.elasticsearch.transport.RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY; @@ -430,17 +430,17 @@ private boolean validateRoleDescriptorsForMixedCluster( listener.onFailure( new IllegalArgumentException( "all nodes must have version [" - + ROLE_REMOTE_CLUSTER_PRIVS + + ROLE_REMOTE_CLUSTER_PRIVS.toReleaseVersion() + "] or higher to support remote cluster privileges for API keys" ) ); return false; } - if (transportVersion.before(ADD_MANAGE_ROLES_PRIVILEGE) && hasGlobalManageRolesPrivilege(roleDescriptors)) { + if (transportVersion.before(TransportVersions.V_8_16_0) && hasGlobalManageRolesPrivilege(roleDescriptors)) { listener.onFailure( new IllegalArgumentException( "all nodes must have version [" - + ADD_MANAGE_ROLES_PRIVILEGE + + TransportVersions.V_8_16_0.toReleaseVersion() + "] or higher to support the manage roles privilege for API keys" ) ); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java index 4ae17a679d205..23a1fc188e4a0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java @@ -481,10 +481,10 @@ private Exception validateRoleDescriptor(RoleDescriptor role) { ); } else if (Arrays.stream(role.getConditionalClusterPrivileges()) .anyMatch(privilege -> privilege instanceof ConfigurableClusterPrivileges.ManageRolesPrivilege) - && clusterService.state().getMinTransportVersion().before(TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE)) { + && clusterService.state().getMinTransportVersion().before(TransportVersions.V_8_16_0)) { return new IllegalStateException( "all nodes must have version [" - + TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE.toReleaseVersion() + + TransportVersions.V_8_16_0.toReleaseVersion() + "] or higher to support the manage roles privilege" ); } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java index ea1b2cdac5a1f..54b7ff6fa484c 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java @@ -158,8 +158,8 @@ public void testRolesWithDescription() throws Exception { public void testRolesWithManageRoles() throws Exception { assumeTrue( - "The manage roles privilege is supported after transport version: " + TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE, - minimumTransportVersion().before(TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE) + "The manage roles privilege is supported after transport version: " + TransportVersions.V_8_16_0, + minimumTransportVersion().before(TransportVersions.V_8_16_0) ); switch (CLUSTER_TYPE) { case OLD -> { @@ -190,7 +190,7 @@ public void testRolesWithManageRoles() throws Exception { } case MIXED -> { try { - this.createClientsByVersion(TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE); + this.createClientsByVersion(TransportVersions.V_8_16_0); // succeed when role manage roles is not provided final String initialRole = randomRoleDescriptorSerialized(); createRole(client(), "my-valid-mixed-role", initialRole); @@ -232,7 +232,7 @@ public void testRolesWithManageRoles() throws Exception { e.getMessage(), containsString( "all nodes must have version [" - + TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE.toReleaseVersion() + + TransportVersions.V_8_16_0.toReleaseVersion() + "] or higher to support the manage roles privilege" ) ); @@ -246,7 +246,7 @@ public void testRolesWithManageRoles() throws Exception { e.getMessage(), containsString( "all nodes must have version [" - + TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE.toReleaseVersion() + + TransportVersions.V_8_16_0.toReleaseVersion() + "] or higher to support the manage roles privilege" ) ); From 724e0524bb56cd38bbe99caf13289eefd1e72793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Thu, 5 Dec 2024 10:29:03 +0100 Subject: [PATCH 039/204] [Entitlements] Integrate PluginsLoader with PolicyManager (#117239) This PR expands `PolicyManager` to actually use `Policy` and `Entitlement` classes for checks, instead of hardcoding them. It also introduces a separate `PluginsResolver`, with a dedicated function to map a Class to a Plugin (name). `PluginsResolver` is initialized with data from `PluginsLoader`, and then its resolve function is used internally in `PolicyManager` to find a plugin policy (and then test against the entitlements declared in the policy). --- .../src/main/java/module-info.java | 1 + .../EntitlementInitialization.java | 8 +- .../api/ElasticsearchEntitlementChecker.java | 13 +- .../policy/CreateClassLoaderEntitlement.java | 1 - ...lementType.java => ExitVMEntitlement.java} | 8 +- .../runtime/policy/FileEntitlement.java | 7 +- .../runtime/policy/PolicyManager.java | 135 +++++++++- .../runtime/policy/PolicyManagerTests.java | 247 +++++++++++++++++ .../bootstrap/Elasticsearch.java | 22 +- .../bootstrap/PluginsResolver.java | 47 ++++ .../elasticsearch/plugins/PluginsLoader.java | 47 +++- .../bootstrap/PluginsResolverTests.java | 254 ++++++++++++++++++ .../plugins/MockPluginsService.java | 2 +- 13 files changed, 738 insertions(+), 54 deletions(-) rename libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/{FlagEntitlementType.java => ExitVMEntitlement.java} (79%) create mode 100644 libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java create mode 100644 server/src/main/java/org/elasticsearch/bootstrap/PluginsResolver.java create mode 100644 server/src/test/java/org/elasticsearch/bootstrap/PluginsResolverTests.java diff --git a/libs/entitlement/src/main/java/module-info.java b/libs/entitlement/src/main/java/module-info.java index 54075ba60bbef..b8a125b98e641 100644 --- a/libs/entitlement/src/main/java/module-info.java +++ b/libs/entitlement/src/main/java/module-info.java @@ -17,6 +17,7 @@ requires static org.elasticsearch.entitlement.bridge; // At runtime, this will be in java.base exports org.elasticsearch.entitlement.runtime.api; + exports org.elasticsearch.entitlement.runtime.policy; exports org.elasticsearch.entitlement.instrumentation; exports org.elasticsearch.entitlement.bootstrap to org.elasticsearch.server; exports org.elasticsearch.entitlement.initialization to java.base; diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index 0ffab5f93969f..fb694308466c6 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -18,6 +18,8 @@ import org.elasticsearch.entitlement.instrumentation.MethodKey; import org.elasticsearch.entitlement.instrumentation.Transformer; import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker; +import org.elasticsearch.entitlement.runtime.policy.CreateClassLoaderEntitlement; +import org.elasticsearch.entitlement.runtime.policy.ExitVMEntitlement; import org.elasticsearch.entitlement.runtime.policy.Policy; import org.elasticsearch.entitlement.runtime.policy.PolicyManager; import org.elasticsearch.entitlement.runtime.policy.PolicyParser; @@ -86,9 +88,11 @@ private static Class internalNameToClass(String internalName) { private static PolicyManager createPolicyManager() throws IOException { Map pluginPolicies = createPluginPolicies(EntitlementBootstrap.bootstrapArgs().pluginData()); - // TODO: What should the name be? // TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it - var serverPolicy = new Policy("server", List.of()); + var serverPolicy = new Policy( + "server", + List.of(new Scope("org.elasticsearch.server", List.of(new ExitVMEntitlement(), new CreateClassLoaderEntitlement()))) + ); return new PolicyManager(serverPolicy, pluginPolicies, EntitlementBootstrap.bootstrapArgs().pluginResolver()); } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index 28a080470c043..aa63b630ed7cd 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -10,7 +10,6 @@ package org.elasticsearch.entitlement.runtime.api; import org.elasticsearch.entitlement.bridge.EntitlementChecker; -import org.elasticsearch.entitlement.runtime.policy.FlagEntitlementType; import org.elasticsearch.entitlement.runtime.policy.PolicyManager; import java.net.URL; @@ -30,27 +29,27 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { @Override public void check$java_lang_System$exit(Class callerClass, int status) { - policyManager.checkFlagEntitlement(callerClass, FlagEntitlementType.SYSTEM_EXIT); + policyManager.checkExitVM(callerClass); } @Override public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls) { - policyManager.checkFlagEntitlement(callerClass, FlagEntitlementType.CREATE_CLASSLOADER); + policyManager.checkCreateClassLoader(callerClass); } @Override public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls, ClassLoader parent) { - policyManager.checkFlagEntitlement(callerClass, FlagEntitlementType.CREATE_CLASSLOADER); + policyManager.checkCreateClassLoader(callerClass); } @Override public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls, ClassLoader parent, URLStreamHandlerFactory factory) { - policyManager.checkFlagEntitlement(callerClass, FlagEntitlementType.CREATE_CLASSLOADER); + policyManager.checkCreateClassLoader(callerClass); } @Override public void check$java_net_URLClassLoader$(Class callerClass, String name, URL[] urls, ClassLoader parent) { - policyManager.checkFlagEntitlement(callerClass, FlagEntitlementType.CREATE_CLASSLOADER); + policyManager.checkCreateClassLoader(callerClass); } @Override @@ -61,6 +60,6 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { ClassLoader parent, URLStreamHandlerFactory factory ) { - policyManager.checkFlagEntitlement(callerClass, FlagEntitlementType.CREATE_CLASSLOADER); + policyManager.checkCreateClassLoader(callerClass); } } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java index 708e0b87711fe..138515be9ffcb 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java @@ -12,5 +12,4 @@ public class CreateClassLoaderEntitlement implements Entitlement { @ExternalEntitlement public CreateClassLoaderEntitlement() {} - } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FlagEntitlementType.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java similarity index 79% rename from libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FlagEntitlementType.java rename to libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java index d40235ee12166..c4a8fc6833581 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FlagEntitlementType.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java @@ -9,7 +9,7 @@ package org.elasticsearch.entitlement.runtime.policy; -public enum FlagEntitlementType { - SYSTEM_EXIT, - CREATE_CLASSLOADER; -} +/** + * Internal policy type (not-parseable -- not available to plugins). + */ +public class ExitVMEntitlement implements Entitlement {} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java index 8df199591d3e4..d0837bc096183 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java @@ -20,6 +20,9 @@ public class FileEntitlement implements Entitlement { public static final int READ_ACTION = 0x1; public static final int WRITE_ACTION = 0x2; + public static final String READ = "read"; + public static final String WRITE = "write"; + private final String path; private final int actions; @@ -29,12 +32,12 @@ public FileEntitlement(String path, List actionsList) { int actionsInt = 0; for (String actionString : actionsList) { - if ("read".equals(actionString)) { + if (READ.equals(actionString)) { if ((actionsInt & READ_ACTION) == READ_ACTION) { throw new IllegalArgumentException("file action [read] specified multiple times"); } actionsInt |= READ_ACTION; - } else if ("write".equals(actionString)) { + } else if (WRITE.equals(actionString)) { if ((actionsInt & WRITE_ACTION) == WRITE_ACTION) { throw new IllegalArgumentException("file action [write] specified multiple times"); } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index b3fb5b75a1d5a..a77c86d5ffd04 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -17,17 +17,45 @@ import java.lang.module.ModuleFinder; import java.lang.module.ModuleReference; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; public class PolicyManager { private static final Logger logger = LogManager.getLogger(ElasticsearchEntitlementChecker.class); + static class ModuleEntitlements { + public static final ModuleEntitlements NONE = new ModuleEntitlements(List.of()); + private final IdentityHashMap, List> entitlementsByType; + + ModuleEntitlements(List entitlements) { + this.entitlementsByType = entitlements.stream() + .collect(Collectors.toMap(Entitlement::getClass, e -> new ArrayList<>(List.of(e)), (a, b) -> { + a.addAll(b); + return a; + }, IdentityHashMap::new)); + } + + public boolean hasEntitlement(Class entitlementClass) { + return entitlementsByType.containsKey(entitlementClass); + } + + public Stream getEntitlements(Class entitlementClass) { + return entitlementsByType.get(entitlementClass).stream().map(entitlementClass::cast); + } + } + + final Map moduleEntitlementsMap = new HashMap<>(); + protected final Policy serverPolicy; protected final Map pluginPolicies; private final Function, String> pluginResolver; @@ -56,27 +84,110 @@ public PolicyManager(Policy defaultPolicy, Map pluginPolicies, F this.pluginResolver = pluginResolver; } - public void checkFlagEntitlement(Class callerClass, FlagEntitlementType type) { + private static List lookupEntitlementsForModule(Policy policy, String moduleName) { + for (int i = 0; i < policy.scopes.size(); ++i) { + var scope = policy.scopes.get(i); + if (scope.name.equals(moduleName)) { + return scope.entitlements; + } + } + return null; + } + + public void checkExitVM(Class callerClass) { + checkEntitlementPresent(callerClass, ExitVMEntitlement.class); + } + + public void checkCreateClassLoader(Class callerClass) { + checkEntitlementPresent(callerClass, CreateClassLoaderEntitlement.class); + } + + private void checkEntitlementPresent(Class callerClass, Class entitlementClass) { var requestingModule = requestingModule(callerClass); if (isTriviallyAllowed(requestingModule)) { return; } - // TODO: real policy check. For now, we only allow our hardcoded System.exit policy for server. - // TODO: this will be checked using policies - if (requestingModule.isNamed() - && requestingModule.getName().equals("org.elasticsearch.server") - && (type == FlagEntitlementType.SYSTEM_EXIT || type == FlagEntitlementType.CREATE_CLASSLOADER)) { - logger.debug("Allowed: caller [{}] in module [{}] has entitlement [{}]", callerClass, requestingModule.getName(), type); + ModuleEntitlements entitlements = getEntitlementsOrThrow(callerClass, requestingModule); + if (entitlements.hasEntitlement(entitlementClass)) { + logger.debug( + () -> Strings.format( + "Entitled: caller [%s], module [%s], type [%s]", + callerClass, + requestingModule.getName(), + entitlementClass.getSimpleName() + ) + ); return; } - - // TODO: plugins policy check using pluginResolver and pluginPolicies throw new NotEntitledException( - Strings.format("Missing entitlement [%s] for caller [%s] in module [%s]", type, callerClass, requestingModule.getName()) + Strings.format( + "Missing entitlement: caller [%s], module [%s], type [%s]", + callerClass, + requestingModule.getName(), + entitlementClass.getSimpleName() + ) ); } + ModuleEntitlements getEntitlementsOrThrow(Class callerClass, Module requestingModule) { + ModuleEntitlements cachedEntitlement = moduleEntitlementsMap.get(requestingModule); + if (cachedEntitlement != null) { + if (cachedEntitlement == ModuleEntitlements.NONE) { + throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, requestingModule) + "[CACHED]"); + } + return cachedEntitlement; + } + + if (isServerModule(requestingModule)) { + var scopeName = requestingModule.getName(); + return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverPolicy, scopeName); + } + + // plugins + var pluginName = pluginResolver.apply(callerClass); + if (pluginName != null) { + var pluginPolicy = pluginPolicies.get(pluginName); + if (pluginPolicy != null) { + final String scopeName; + if (requestingModule.isNamed() == false) { + scopeName = ALL_UNNAMED; + } else { + scopeName = requestingModule.getName(); + } + return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginPolicy, scopeName); + } + } + + moduleEntitlementsMap.put(requestingModule, ModuleEntitlements.NONE); + throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, requestingModule)); + } + + private static String buildModuleNoPolicyMessage(Class callerClass, Module requestingModule) { + return Strings.format("Missing entitlement policy: caller [%s], module [%s]", callerClass, requestingModule.getName()); + } + + private ModuleEntitlements getModuleEntitlementsOrThrow(Class callerClass, Module module, Policy policy, String moduleName) { + var entitlements = lookupEntitlementsForModule(policy, moduleName); + if (entitlements == null) { + // Module without entitlements - remember we don't have any + moduleEntitlementsMap.put(module, ModuleEntitlements.NONE); + throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, module)); + } + // We have a policy for this module + var classEntitlements = createClassEntitlements(entitlements); + moduleEntitlementsMap.put(module, classEntitlements); + return classEntitlements; + } + + private static boolean isServerModule(Module requestingModule) { + return requestingModule.isNamed() && requestingModule.getLayer() == ModuleLayer.boot(); + } + + private ModuleEntitlements createClassEntitlements(List entitlements) { + return new ModuleEntitlements(entitlements); + } + private static Module requestingModule(Class callerClass) { if (callerClass != null) { Module callerModule = callerClass.getModule(); @@ -102,10 +213,10 @@ private static Module requestingModule(Class callerClass) { private static boolean isTriviallyAllowed(Module requestingModule) { if (requestingModule == null) { - logger.debug("Trivially allowed: entire call stack is in composed of classes in system modules"); + logger.debug("Entitlement trivially allowed: entire call stack is in composed of classes in system modules"); return true; } - logger.trace("Not trivially allowed"); + logger.trace("Entitlement not trivially allowed"); return false; } diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java new file mode 100644 index 0000000000000..45bdf2e457824 --- /dev/null +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java @@ -0,0 +1,247 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.entitlement.runtime.api.NotEntitledException; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.compiler.InMemoryJavaCompiler; +import org.elasticsearch.test.jar.JarUtils; + +import java.io.IOException; +import java.lang.module.Configuration; +import java.lang.module.ModuleFinder; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static java.util.Map.entry; +import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ALL_UNNAMED; +import static org.elasticsearch.test.LambdaMatchers.transformedMatch; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.sameInstance; + +@ESTestCase.WithoutSecurityManager +public class PolicyManagerTests extends ESTestCase { + + public void testGetEntitlementsThrowsOnMissingPluginUnnamedModule() { + var policyManager = new PolicyManager( + createEmptyTestServerPolicy(), + Map.of("plugin1", createPluginPolicy("plugin.module")), + c -> "plugin1" + ); + + // Any class from the current module (unnamed) will do + var callerClass = this.getClass(); + var requestingModule = callerClass.getModule(); + + var ex = assertThrows( + "No policy for the unnamed module", + NotEntitledException.class, + () -> policyManager.getEntitlementsOrThrow(callerClass, requestingModule) + ); + + assertEquals( + "Missing entitlement policy: caller [class org.elasticsearch.entitlement.runtime.policy.PolicyManagerTests], module [null]", + ex.getMessage() + ); + assertThat(policyManager.moduleEntitlementsMap, hasEntry(requestingModule, PolicyManager.ModuleEntitlements.NONE)); + } + + public void testGetEntitlementsThrowsOnMissingPolicyForPlugin() { + var policyManager = new PolicyManager(createEmptyTestServerPolicy(), Map.of(), c -> "plugin1"); + + // Any class from the current module (unnamed) will do + var callerClass = this.getClass(); + var requestingModule = callerClass.getModule(); + + var ex = assertThrows( + "No policy for this plugin", + NotEntitledException.class, + () -> policyManager.getEntitlementsOrThrow(callerClass, requestingModule) + ); + + assertEquals( + "Missing entitlement policy: caller [class org.elasticsearch.entitlement.runtime.policy.PolicyManagerTests], module [null]", + ex.getMessage() + ); + assertThat(policyManager.moduleEntitlementsMap, hasEntry(requestingModule, PolicyManager.ModuleEntitlements.NONE)); + } + + public void testGetEntitlementsFailureIsCached() { + var policyManager = new PolicyManager(createEmptyTestServerPolicy(), Map.of(), c -> "plugin1"); + + // Any class from the current module (unnamed) will do + var callerClass = this.getClass(); + var requestingModule = callerClass.getModule(); + + assertThrows(NotEntitledException.class, () -> policyManager.getEntitlementsOrThrow(callerClass, requestingModule)); + assertThat(policyManager.moduleEntitlementsMap, hasEntry(requestingModule, PolicyManager.ModuleEntitlements.NONE)); + + // A second time + var ex = assertThrows(NotEntitledException.class, () -> policyManager.getEntitlementsOrThrow(callerClass, requestingModule)); + + assertThat(ex.getMessage(), endsWith("[CACHED]")); + // Nothing new in the map + assertThat(policyManager.moduleEntitlementsMap, aMapWithSize(1)); + } + + public void testGetEntitlementsReturnsEntitlementsForPluginUnnamedModule() { + var policyManager = new PolicyManager( + createEmptyTestServerPolicy(), + Map.ofEntries(entry("plugin2", createPluginPolicy(ALL_UNNAMED))), + c -> "plugin2" + ); + + // Any class from the current module (unnamed) will do + var callerClass = this.getClass(); + var requestingModule = callerClass.getModule(); + + var entitlements = policyManager.getEntitlementsOrThrow(callerClass, requestingModule); + assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); + } + + public void testGetEntitlementsThrowsOnMissingPolicyForServer() throws ClassNotFoundException { + var policyManager = new PolicyManager(createTestServerPolicy("example"), Map.of(), c -> null); + + // Tests do not run modular, so we cannot use a server class. + // But we know that in production code the server module and its classes are in the boot layer. + // So we use a random module in the boot layer, and a random class from that module (not java.base -- it is + // loaded too early) to mimic a class that would be in the server module. + var mockServerClass = ModuleLayer.boot().findLoader("jdk.httpserver").loadClass("com.sun.net.httpserver.HttpServer"); + var requestingModule = mockServerClass.getModule(); + + var ex = assertThrows( + "No policy for this module in server", + NotEntitledException.class, + () -> policyManager.getEntitlementsOrThrow(mockServerClass, requestingModule) + ); + + assertEquals( + "Missing entitlement policy: caller [class com.sun.net.httpserver.HttpServer], module [jdk.httpserver]", + ex.getMessage() + ); + assertThat(policyManager.moduleEntitlementsMap, hasEntry(requestingModule, PolicyManager.ModuleEntitlements.NONE)); + } + + public void testGetEntitlementsReturnsEntitlementsForServerModule() throws ClassNotFoundException { + var policyManager = new PolicyManager(createTestServerPolicy("jdk.httpserver"), Map.of(), c -> null); + + // Tests do not run modular, so we cannot use a server class. + // But we know that in production code the server module and its classes are in the boot layer. + // So we use a random module in the boot layer, and a random class from that module (not java.base -- it is + // loaded too early) to mimic a class that would be in the server module. + var mockServerClass = ModuleLayer.boot().findLoader("jdk.httpserver").loadClass("com.sun.net.httpserver.HttpServer"); + var requestingModule = mockServerClass.getModule(); + + var entitlements = policyManager.getEntitlementsOrThrow(mockServerClass, requestingModule); + assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); + assertThat(entitlements.hasEntitlement(ExitVMEntitlement.class), is(true)); + } + + public void testGetEntitlementsReturnsEntitlementsForPluginModule() throws IOException, ClassNotFoundException { + final Path home = createTempDir(); + + Path jar = creteMockPluginJar(home); + + var policyManager = new PolicyManager( + createEmptyTestServerPolicy(), + Map.of("mock-plugin", createPluginPolicy("org.example.plugin")), + c -> "mock-plugin" + ); + + var layer = createLayerForJar(jar, "org.example.plugin"); + var mockPluginClass = layer.findLoader("org.example.plugin").loadClass("q.B"); + var requestingModule = mockPluginClass.getModule(); + + var entitlements = policyManager.getEntitlementsOrThrow(mockPluginClass, requestingModule); + assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); + assertThat( + entitlements.getEntitlements(FileEntitlement.class).toList(), + contains(transformedMatch(FileEntitlement::toString, containsString("/test/path"))) + ); + } + + public void testGetEntitlementsResultIsCached() { + var policyManager = new PolicyManager( + createEmptyTestServerPolicy(), + Map.ofEntries(entry("plugin2", createPluginPolicy(ALL_UNNAMED))), + c -> "plugin2" + ); + + // Any class from the current module (unnamed) will do + var callerClass = this.getClass(); + var requestingModule = callerClass.getModule(); + + var entitlements = policyManager.getEntitlementsOrThrow(callerClass, requestingModule); + assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); + assertThat(policyManager.moduleEntitlementsMap, aMapWithSize(1)); + var cachedResult = policyManager.moduleEntitlementsMap.values().stream().findFirst().get(); + var entitlementsAgain = policyManager.getEntitlementsOrThrow(callerClass, requestingModule); + + // Nothing new in the map + assertThat(policyManager.moduleEntitlementsMap, aMapWithSize(1)); + assertThat(entitlementsAgain, sameInstance(cachedResult)); + } + + private static Policy createEmptyTestServerPolicy() { + return new Policy("server", List.of()); + } + + private static Policy createTestServerPolicy(String scopeName) { + return new Policy("server", List.of(new Scope(scopeName, List.of(new ExitVMEntitlement(), new CreateClassLoaderEntitlement())))); + } + + private static Policy createPluginPolicy(String... pluginModules) { + return new Policy( + "plugin", + Arrays.stream(pluginModules) + .map( + name -> new Scope( + name, + List.of(new FileEntitlement("/test/path", List.of(FileEntitlement.READ)), new CreateClassLoaderEntitlement()) + ) + ) + .toList() + ); + } + + private static Path creteMockPluginJar(Path home) throws IOException { + Path jar = home.resolve("mock-plugin.jar"); + + Map sources = Map.ofEntries( + entry("module-info", "module org.example.plugin { exports q; }"), + entry("q.B", "package q; public class B { }") + ); + + var classToBytes = InMemoryJavaCompiler.compile(sources); + JarUtils.createJarWithEntries( + jar, + Map.ofEntries(entry("module-info.class", classToBytes.get("module-info")), entry("q/B.class", classToBytes.get("q.B"))) + ); + return jar; + } + + private static ModuleLayer createLayerForJar(Path jar, String moduleName) { + Configuration cf = ModuleLayer.boot().configuration().resolve(ModuleFinder.of(jar), ModuleFinder.of(), Set.of(moduleName)); + var moduleController = ModuleLayer.defineModulesWithOneLoader( + cf, + List.of(ModuleLayer.boot()), + ClassLoader.getPlatformClassLoader() + ); + return moduleController.layer(); + } +} diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index c06ea9305aef8..27cbb39c05d38 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -42,9 +42,7 @@ import org.elasticsearch.nativeaccess.NativeAccess; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; -import org.elasticsearch.plugins.PluginBundle; import org.elasticsearch.plugins.PluginsLoader; -import org.elasticsearch.plugins.PluginsUtils; import java.io.IOException; import java.io.InputStream; @@ -54,10 +52,8 @@ import java.nio.file.Path; import java.security.Permission; import java.security.Security; -import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -208,21 +204,17 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException { // load the plugin Java modules and layers now for use in entitlements var pluginsLoader = PluginsLoader.createPluginsLoader(nodeEnv.modulesFile(), nodeEnv.pluginsFile()); bootstrap.setPluginsLoader(pluginsLoader); + var pluginsResolver = PluginsResolver.create(pluginsLoader); if (Boolean.parseBoolean(System.getProperty("es.entitlements.enabled"))) { LogManager.getLogger(Elasticsearch.class).info("Bootstrapping Entitlements"); - List> pluginData = new ArrayList<>(); - Set moduleBundles = PluginsUtils.getModuleBundles(nodeEnv.modulesFile()); - for (PluginBundle moduleBundle : moduleBundles) { - pluginData.add(Tuple.tuple(moduleBundle.getDir(), moduleBundle.pluginDescriptor().isModular())); - } - Set pluginBundles = PluginsUtils.getPluginBundles(nodeEnv.pluginsFile()); - for (PluginBundle pluginBundle : pluginBundles) { - pluginData.add(Tuple.tuple(pluginBundle.getDir(), pluginBundle.pluginDescriptor().isModular())); - } - // TODO: add a functor to map module to plugin name - EntitlementBootstrap.bootstrap(pluginData, callerClass -> null); + List> pluginData = pluginsLoader.allBundles() + .stream() + .map(bundle -> Tuple.tuple(bundle.getDir(), bundle.pluginDescriptor().isModular())) + .toList(); + + EntitlementBootstrap.bootstrap(pluginData, pluginsResolver::resolveClassToPluginName); } else { // install SM after natives, shutdown hooks, etc. LogManager.getLogger(Elasticsearch.class).info("Bootstrapping java SecurityManager"); diff --git a/server/src/main/java/org/elasticsearch/bootstrap/PluginsResolver.java b/server/src/main/java/org/elasticsearch/bootstrap/PluginsResolver.java new file mode 100644 index 0000000000000..256e91cbee16d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/bootstrap/PluginsResolver.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.bootstrap; + +import org.elasticsearch.plugins.PluginsLoader; + +import java.util.HashMap; +import java.util.Map; + +class PluginsResolver { + private final Map pluginNameByModule; + + private PluginsResolver(Map pluginNameByModule) { + this.pluginNameByModule = pluginNameByModule; + } + + public static PluginsResolver create(PluginsLoader pluginsLoader) { + Map pluginNameByModule = new HashMap<>(); + + pluginsLoader.pluginLayers().forEach(pluginLayer -> { + var pluginName = pluginLayer.pluginBundle().pluginDescriptor().getName(); + if (pluginLayer.pluginModuleLayer() != null && pluginLayer.pluginModuleLayer() != ModuleLayer.boot()) { + // This plugin is a Java Module + for (var module : pluginLayer.pluginModuleLayer().modules()) { + pluginNameByModule.put(module, pluginName); + } + } else { + // This plugin is not modularized + pluginNameByModule.put(pluginLayer.pluginClassLoader().getUnnamedModule(), pluginName); + } + }); + + return new PluginsResolver(pluginNameByModule); + } + + public String resolveClassToPluginName(Class clazz) { + var module = clazz.getModule(); + return pluginNameByModule.get(module); + } +} diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java b/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java index aa21e5c64d903..aadda93f977b6 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java @@ -50,7 +50,6 @@ * to have all the plugin information they need prior to starting. */ public class PluginsLoader { - /** * Contains information about the {@link ClassLoader} required to load a plugin */ @@ -64,18 +63,26 @@ public interface PluginLayer { * @return The {@link ClassLoader} used to instantiate the main class for the plugin */ ClassLoader pluginClassLoader(); + + /** + * @return The {@link ModuleLayer} for the plugin modules + */ + ModuleLayer pluginModuleLayer(); } /** * Contains information about the {@link ClassLoader}s and {@link ModuleLayer} required for loading a plugin - * @param pluginBundle Information about the bundle of jars used in this plugin + * + * @param pluginBundle Information about the bundle of jars used in this plugin * @param pluginClassLoader The {@link ClassLoader} used to instantiate the main class for the plugin - * @param spiClassLoader The exported {@link ClassLoader} visible to other Java modules - * @param spiModuleLayer The exported {@link ModuleLayer} visible to other Java modules + * @param pluginModuleLayer The {@link ModuleLayer} containing the Java modules of the plugin + * @param spiClassLoader The exported {@link ClassLoader} visible to other Java modules + * @param spiModuleLayer The exported {@link ModuleLayer} visible to other Java modules */ private record LoadedPluginLayer( PluginBundle pluginBundle, ClassLoader pluginClassLoader, + ModuleLayer pluginModuleLayer, ClassLoader spiClassLoader, ModuleLayer spiModuleLayer ) implements PluginLayer { @@ -103,6 +110,10 @@ public record LayerAndLoader(ModuleLayer layer, ClassLoader loader) { public static LayerAndLoader ofLoader(ClassLoader loader) { return new LayerAndLoader(ModuleLayer.boot(), loader); } + + public static LayerAndLoader ofUberModuleLoader(UberModuleClassLoader loader) { + return new LayerAndLoader(loader.getLayer(), loader); + } } private static final Logger logger = LogManager.getLogger(PluginsLoader.class); @@ -111,6 +122,7 @@ public static LayerAndLoader ofLoader(ClassLoader loader) { private final List moduleDescriptors; private final List pluginDescriptors; private final Map loadedPluginLayers; + private final Set allBundles; /** * Constructs a new PluginsLoader @@ -185,17 +197,19 @@ public static PluginsLoader createPluginsLoader(Path modulesDirectory, Path plug } } - return new PluginsLoader(moduleDescriptors, pluginDescriptors, loadedPluginLayers); + return new PluginsLoader(moduleDescriptors, pluginDescriptors, loadedPluginLayers, Set.copyOf(seenBundles)); } PluginsLoader( List moduleDescriptors, List pluginDescriptors, - Map loadedPluginLayers + Map loadedPluginLayers, + Set allBundles ) { this.moduleDescriptors = moduleDescriptors; this.pluginDescriptors = pluginDescriptors; this.loadedPluginLayers = loadedPluginLayers; + this.allBundles = allBundles; } public List moduleDescriptors() { @@ -210,6 +224,10 @@ public Stream pluginLayers() { return loadedPluginLayers.values().stream().map(Function.identity()); } + public Set allBundles() { + return allBundles; + } + private static void loadPluginLayer( PluginBundle bundle, Map loaded, @@ -239,7 +257,7 @@ private static void loadPluginLayer( } final ClassLoader pluginParentLoader = spiLayerAndLoader == null ? parentLoader : spiLayerAndLoader.loader(); - final LayerAndLoader pluginLayerAndLoader = createPlugin( + final LayerAndLoader pluginLayerAndLoader = createPluginLayerAndLoader( bundle, pluginParentLoader, extendedPlugins, @@ -253,7 +271,16 @@ private static void loadPluginLayer( spiLayerAndLoader = pluginLayerAndLoader; } - loaded.put(name, new LoadedPluginLayer(bundle, pluginClassLoader, spiLayerAndLoader.loader, spiLayerAndLoader.layer)); + loaded.put( + name, + new LoadedPluginLayer( + bundle, + pluginClassLoader, + pluginLayerAndLoader.layer(), + spiLayerAndLoader.loader, + spiLayerAndLoader.layer + ) + ); } static LayerAndLoader createSPI( @@ -277,7 +304,7 @@ static LayerAndLoader createSPI( } } - static LayerAndLoader createPlugin( + private static LayerAndLoader createPluginLayerAndLoader( PluginBundle bundle, ClassLoader pluginParentLoader, List extendedPlugins, @@ -294,7 +321,7 @@ static LayerAndLoader createPlugin( return createPluginModuleLayer(bundle, pluginParentLoader, parentLayers, qualifiedExports); } else if (plugin.isStable()) { logger.debug(() -> "Loading bundle: " + plugin.getName() + ", non-modular as synthetic module"); - return LayerAndLoader.ofLoader( + return LayerAndLoader.ofUberModuleLoader( UberModuleClassLoader.getInstance( pluginParentLoader, ModuleLayer.boot(), diff --git a/server/src/test/java/org/elasticsearch/bootstrap/PluginsResolverTests.java b/server/src/test/java/org/elasticsearch/bootstrap/PluginsResolverTests.java new file mode 100644 index 0000000000000..331f0f7ad13e9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/bootstrap/PluginsResolverTests.java @@ -0,0 +1,254 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.bootstrap; + +import org.elasticsearch.plugins.PluginBundle; +import org.elasticsearch.plugins.PluginDescriptor; +import org.elasticsearch.plugins.PluginsLoader; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.compiler.InMemoryJavaCompiler; +import org.elasticsearch.test.jar.JarUtils; + +import java.io.IOException; +import java.lang.module.Configuration; +import java.lang.module.ModuleFinder; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static java.util.Map.entry; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ESTestCase.WithoutSecurityManager +public class PluginsResolverTests extends ESTestCase { + + private record TestPluginLayer(PluginBundle pluginBundle, ClassLoader pluginClassLoader, ModuleLayer pluginModuleLayer) + implements + PluginsLoader.PluginLayer {} + + public void testResolveModularPlugin() throws IOException, ClassNotFoundException { + String moduleName = "modular.plugin"; + String pluginName = "modular-plugin"; + + final Path home = createTempDir(); + + Path jar = createModularPluginJar(home, pluginName, moduleName, "p", "A"); + + var layer = createModuleLayer(moduleName, jar); + var loader = layer.findLoader(moduleName); + + PluginBundle bundle = createMockBundle(pluginName, moduleName, "p.A"); + PluginsLoader mockPluginsLoader = mock(PluginsLoader.class); + + when(mockPluginsLoader.pluginLayers()).thenReturn(Stream.of(new TestPluginLayer(bundle, loader, layer))); + PluginsResolver pluginsResolver = PluginsResolver.create(mockPluginsLoader); + + var testClass = loader.loadClass("p.A"); + var resolvedPluginName = pluginsResolver.resolveClassToPluginName(testClass); + var unresolvedPluginName1 = pluginsResolver.resolveClassToPluginName(PluginsResolver.class); + var unresolvedPluginName2 = pluginsResolver.resolveClassToPluginName(String.class); + + assertEquals(pluginName, resolvedPluginName); + assertNull(unresolvedPluginName1); + assertNull(unresolvedPluginName2); + } + + public void testResolveMultipleModularPlugins() throws IOException, ClassNotFoundException { + final Path home = createTempDir(); + + Path jar1 = createModularPluginJar(home, "plugin1", "module.one", "p", "A"); + Path jar2 = createModularPluginJar(home, "plugin2", "module.two", "q", "B"); + + var layer1 = createModuleLayer("module.one", jar1); + var loader1 = layer1.findLoader("module.one"); + var layer2 = createModuleLayer("module.two", jar2); + var loader2 = layer2.findLoader("module.two"); + + PluginBundle bundle1 = createMockBundle("plugin1", "module.one", "p.A"); + PluginBundle bundle2 = createMockBundle("plugin2", "module.two", "q.B"); + PluginsLoader mockPluginsLoader = mock(PluginsLoader.class); + + when(mockPluginsLoader.pluginLayers()).thenReturn( + Stream.of(new TestPluginLayer(bundle1, loader1, layer1), new TestPluginLayer(bundle2, loader2, layer2)) + ); + PluginsResolver pluginsResolver = PluginsResolver.create(mockPluginsLoader); + + var testClass1 = loader1.loadClass("p.A"); + var testClass2 = loader2.loadClass("q.B"); + var resolvedPluginName1 = pluginsResolver.resolveClassToPluginName(testClass1); + var resolvedPluginName2 = pluginsResolver.resolveClassToPluginName(testClass2); + + assertEquals("plugin1", resolvedPluginName1); + assertEquals("plugin2", resolvedPluginName2); + } + + public void testResolveReferencedModulesInModularPlugins() throws IOException, ClassNotFoundException { + final Path home = createTempDir(); + + Path dependencyJar = createModularPluginJar(home, "plugin1", "module.one", "p", "A"); + Path pluginJar = home.resolve("plugin2.jar"); + + Map sources = Map.ofEntries( + entry("module-info", "module module.two { exports q; requires module.one; }"), + entry("q.B", "package q; public class B { public p.A a = null; }") + ); + + var classToBytes = InMemoryJavaCompiler.compile(sources, "--add-modules", "module.one", "-p", home.toString()); + JarUtils.createJarWithEntries( + pluginJar, + Map.ofEntries(entry("module-info.class", classToBytes.get("module-info")), entry("q/B.class", classToBytes.get("q.B"))) + ); + + var layer = createModuleLayer("module.two", pluginJar, dependencyJar); + var loader = layer.findLoader("module.two"); + + PluginBundle bundle = createMockBundle("plugin2", "module.two", "q.B"); + PluginsLoader mockPluginsLoader = mock(PluginsLoader.class); + + when(mockPluginsLoader.pluginLayers()).thenReturn(Stream.of(new TestPluginLayer(bundle, loader, layer))); + PluginsResolver pluginsResolver = PluginsResolver.create(mockPluginsLoader); + + var testClass1 = loader.loadClass("p.A"); + var testClass2 = loader.loadClass("q.B"); + var resolvedPluginName1 = pluginsResolver.resolveClassToPluginName(testClass1); + var resolvedPluginName2 = pluginsResolver.resolveClassToPluginName(testClass2); + + assertEquals("plugin2", resolvedPluginName1); + assertEquals("plugin2", resolvedPluginName2); + } + + public void testResolveMultipleNonModularPlugins() throws IOException, ClassNotFoundException { + final Path home = createTempDir(); + + Path jar1 = createNonModularPluginJar(home, "plugin1", "p", "A"); + Path jar2 = createNonModularPluginJar(home, "plugin2", "q", "B"); + + var loader1 = createClassLoader(jar1); + var loader2 = createClassLoader(jar2); + + PluginBundle bundle1 = createMockBundle("plugin1", null, "p.A"); + PluginBundle bundle2 = createMockBundle("plugin2", null, "q.B"); + PluginsLoader mockPluginsLoader = mock(PluginsLoader.class); + + when(mockPluginsLoader.pluginLayers()).thenReturn( + Stream.of(new TestPluginLayer(bundle1, loader1, ModuleLayer.boot()), new TestPluginLayer(bundle2, loader2, ModuleLayer.boot())) + ); + PluginsResolver pluginsResolver = PluginsResolver.create(mockPluginsLoader); + + var testClass1 = loader1.loadClass("p.A"); + var testClass2 = loader2.loadClass("q.B"); + var resolvedPluginName1 = pluginsResolver.resolveClassToPluginName(testClass1); + var resolvedPluginName2 = pluginsResolver.resolveClassToPluginName(testClass2); + + assertEquals("plugin1", resolvedPluginName1); + assertEquals("plugin2", resolvedPluginName2); + } + + public void testResolveNonModularPlugin() throws IOException, ClassNotFoundException { + String pluginName = "non-modular-plugin"; + + final Path home = createTempDir(); + + Path jar = createNonModularPluginJar(home, pluginName, "p", "A"); + + var loader = createClassLoader(jar); + + PluginBundle bundle = createMockBundle(pluginName, null, "p.A"); + PluginsLoader mockPluginsLoader = mock(PluginsLoader.class); + + when(mockPluginsLoader.pluginLayers()).thenReturn(Stream.of(new TestPluginLayer(bundle, loader, ModuleLayer.boot()))); + PluginsResolver pluginsResolver = PluginsResolver.create(mockPluginsLoader); + + var testClass = loader.loadClass("p.A"); + var resolvedPluginName = pluginsResolver.resolveClassToPluginName(testClass); + var unresolvedPluginName1 = pluginsResolver.resolveClassToPluginName(PluginsResolver.class); + var unresolvedPluginName2 = pluginsResolver.resolveClassToPluginName(String.class); + + assertEquals(pluginName, resolvedPluginName); + assertNull(unresolvedPluginName1); + assertNull(unresolvedPluginName2); + } + + private static URLClassLoader createClassLoader(Path jar) throws MalformedURLException { + return new URLClassLoader(new URL[] { jar.toUri().toURL() }); + } + + private static ModuleLayer createModuleLayer(String moduleName, Path... jars) { + var finder = ModuleFinder.of(jars); + Configuration cf = ModuleLayer.boot().configuration().resolve(finder, ModuleFinder.of(), Set.of(moduleName)); + var moduleController = ModuleLayer.defineModulesWithOneLoader( + cf, + List.of(ModuleLayer.boot()), + ClassLoader.getPlatformClassLoader() + ); + return moduleController.layer(); + } + + private static PluginBundle createMockBundle(String pluginName, String moduleName, String fqClassName) { + PluginDescriptor pd = new PluginDescriptor( + pluginName, + null, + null, + null, + null, + fqClassName, + moduleName, + List.of(), + false, + false, + true, + false + ); + + PluginBundle bundle = mock(PluginBundle.class); + when(bundle.pluginDescriptor()).thenReturn(pd); + return bundle; + } + + private static Path createModularPluginJar(Path home, String pluginName, String moduleName, String packageName, String className) + throws IOException { + Path jar = home.resolve(pluginName + ".jar"); + String fqClassName = packageName + "." + className; + + Map sources = Map.ofEntries( + entry("module-info", "module " + moduleName + " { exports " + packageName + "; }"), + entry(fqClassName, "package " + packageName + "; public class " + className + " {}") + ); + + var classToBytes = InMemoryJavaCompiler.compile(sources); + JarUtils.createJarWithEntries( + jar, + Map.ofEntries( + entry("module-info.class", classToBytes.get("module-info")), + entry(packageName + "/" + className + ".class", classToBytes.get(fqClassName)) + ) + ); + return jar; + } + + private static Path createNonModularPluginJar(Path home, String pluginName, String packageName, String className) throws IOException { + Path jar = home.resolve(pluginName + ".jar"); + String fqClassName = packageName + "." + className; + + Map sources = Map.ofEntries( + entry(fqClassName, "package " + packageName + "; public class " + className + " {}") + ); + + var classToBytes = InMemoryJavaCompiler.compile(sources); + JarUtils.createJarWithEntries(jar, Map.ofEntries(entry(packageName + "/" + className + ".class", classToBytes.get(fqClassName)))); + return jar; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java index a9a825af3b865..91875600ec000 100644 --- a/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java +++ b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java @@ -45,7 +45,7 @@ public MockPluginsService(Settings settings, Environment environment, Collection super( settings, environment.configFile(), - new PluginsLoader(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap()) + new PluginsLoader(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), Collections.emptySet()) ); List pluginsLoaded = new ArrayList<>(); From 14d9e7c1f5d8ba7a2d30f1a28735c3bf803abc15 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 5 Dec 2024 11:24:27 +0100 Subject: [PATCH 040/204] ES|QL: Fix ESQL usage output (telemetry) non-snapshot version test (#118051) --- muted-tests.yml | 3 --- .../resources/rest-api-spec/test/esql/60_usage.yml | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index de18c08337e11..f525607a848d0 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -248,9 +248,6 @@ tests: - class: org.elasticsearch.index.reindex.ReindexNodeShutdownIT method: testReindexWithShutdown issue: https://github.com/elastic/elasticsearch/issues/118040 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=esql/60_usage/Basic ESQL usage output (telemetry) non-snapshot version} - issue: https://github.com/elastic/elasticsearch/issues/117862 # Examples: # diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index 26e3c8ed0ef47..e6c061f44a9e4 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -163,4 +163,4 @@ setup: - match: {esql.functions.cos: $functions_cos} - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} - - length: {esql.functions: 123} # check the "sister" test above for a likely update to the same esql.functions length check + - length: {esql.functions: 124} # check the "sister" test above for a likely update to the same esql.functions length check From e4defcaec3f2433fda2d0187543417e679570b37 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 5 Dec 2024 11:43:56 +0100 Subject: [PATCH 041/204] ESQL: Small LOOKUP JOIN cleanups (#117922) * Simplify InsertFieldExtraction * Fix wrong error message when looping ResolveRefs * Add basic verification tests * Add check for data type mismatch --- .../src/main/resources/lookup-join.csv-spec | 31 ++++++++++ .../xpack/esql/analysis/Analyzer.java | 9 +-- .../xpack/esql/analysis/AnalyzerContext.java | 8 ++- .../xpack/esql/analysis/Verifier.java | 48 ++++++++++------ .../physical/local/InsertFieldExtraction.java | 20 ++----- .../esql/planner/LocalExecutionPlanner.java | 9 --- .../xpack/esql/session/EsqlSession.java | 3 +- .../esql/analysis/AnalyzerTestUtils.java | 29 ++++++++-- .../xpack/esql/analysis/AnalyzerTests.java | 56 +++++++++++++++++++ .../xpack/esql/analysis/VerifierTests.java | 11 ++++ 10 files changed, 173 insertions(+), 51 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 712cadf6d44fd..584cde55080ef 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -120,6 +120,19 @@ left:keyword | client_ip:keyword | right:keyword | env:keyword left | 172.21.0.5 | right | Development ; +lookupIPFromRowWithShadowingKeepReordered +required_capability: join_lookup_v4 + +ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" +| EVAL client_ip = client_ip::keyword +| LOOKUP JOIN clientips_lookup ON client_ip +| KEEP right, env, client_ip +; + +right:keyword | env:keyword | client_ip:keyword +right | Development | 172.21.0.5 +; + lookupIPFromIndex required_capability: join_lookup_v4 @@ -263,6 +276,24 @@ ignoreOrder:true; 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | Success ; +lookupMessageFromIndexKeepReordered +required_capability: join_lookup_v4 + +FROM sample_data +| LOOKUP JOIN message_types_lookup ON message +| KEEP type, client_ip, event_duration, message +; + +type:keyword | client_ip:ip | event_duration:long | message:keyword +Success | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +Error | 172.21.3.15 | 5033755 | Connection error +Error | 172.21.3.15 | 8268153 | Connection error +Error | 172.21.3.15 | 725448 | Connection error +Disconnected | 172.21.0.5 | 1232382 | Disconnected +Success | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +Success | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + lookupMessageFromIndexStats required_capability: join_lookup_v4 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index b847508d2b161..cf91c7df9a034 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -633,9 +633,10 @@ private Join resolveLookupJoin(LookupJoin join) { config = new JoinConfig(coreJoin, leftKeys, leftKeys, rightKeys); join = new LookupJoin(join.source(), join.left(), join.right(), config); - } - // everything else is unsupported for now - else { + } else if (type != JoinTypes.LEFT) { + // everything else is unsupported for now + // LEFT can only happen by being mapped from a USING above. So we need to exclude this as well because this rule can be run + // more than once. UnresolvedAttribute errorAttribute = new UnresolvedAttribute(join.source(), "unsupported", "Unsupported join type"); // add error message return join.withConfig(new JoinConfig(type, singletonList(errorAttribute), emptyList(), emptyList())); @@ -651,7 +652,7 @@ private List resolveUsingColumns(List cols, List"), enrichResolution); + this( + configuration, + functionRegistry, + indexResolution, + IndexResolution.invalid("AnalyzerContext constructed without any lookup join resolution"), + enrichResolution + ); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 49d8a5ee8caad..f5fd82d742bc7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.analysis; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.common.Failure; @@ -55,7 +54,8 @@ import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; -import org.elasticsearch.xpack.esql.plan.logical.join.LookupJoin; +import org.elasticsearch.xpack.esql.plan.logical.join.Join; +import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig; import org.elasticsearch.xpack.esql.stats.FeatureMetric; import org.elasticsearch.xpack.esql.stats.Metrics; @@ -172,20 +172,6 @@ else if (p instanceof Lookup lookup) { else { lookup.matchFields().forEach(unresolvedExpressions); } - } else if (p instanceof LookupJoin lj) { - // expect right side to always be a lookup index - lj.right().forEachUp(EsRelation.class, r -> { - if (r.indexMode() != IndexMode.LOOKUP) { - failures.add( - fail( - r, - "LOOKUP JOIN right side [{}] must be a lookup index (index_mode=lookup, not [{}]", - r.index().name(), - r.indexMode().getName() - ) - ); - } - }); } else { @@ -217,6 +203,7 @@ else if (p instanceof Lookup lookup) { checkSort(p, failures); checkFullTextQueryFunctions(p, failures); + checkJoin(p, failures); }); checkRemoteEnrich(plan, failures); checkMetadataScoreNameReserved(plan, failures); @@ -791,6 +778,35 @@ private static void checkNotPresentInDisjunctions( }); } + /** + * Checks Joins for invalid usage. + * + * @param plan root plan to check + * @param failures failures found + */ + private static void checkJoin(LogicalPlan plan, Set failures) { + if (plan instanceof Join join) { + JoinConfig config = join.config(); + for (int i = 0; i < config.leftFields().size(); i++) { + Attribute leftField = config.leftFields().get(i); + Attribute rightField = config.rightFields().get(i); + if (leftField.dataType() != rightField.dataType()) { + failures.add( + fail( + leftField, + "JOIN left field [{}] of type [{}] is incompatible with right field [{}] of type [{}]", + leftField.name(), + leftField.dataType(), + rightField.name(), + rightField.dataType() + ) + ); + } + } + + } + } + /** * Checks full text query functions for invalid usage. * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/InsertFieldExtraction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/InsertFieldExtraction.java index dc32a4ad3c282..ed8851b64c27e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/InsertFieldExtraction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/InsertFieldExtraction.java @@ -11,14 +11,12 @@ import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; -import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.optimizer.rules.physical.ProjectAwayColumns; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec; import org.elasticsearch.xpack.esql.plan.physical.LeafExec; -import org.elasticsearch.xpack.esql.plan.physical.LookupJoinExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.rule.Rule; @@ -102,25 +100,17 @@ private static Set missingAttributes(PhysicalPlan p) { var missing = new LinkedHashSet(); var input = p.inputSet(); - // For LOOKUP JOIN we only need field-extraction on left fields used to match, since the right side is always materialized - if (p instanceof LookupJoinExec join) { - join.leftFields().forEach(f -> { - if (input.contains(f) == false) { - missing.add(f); - } - }); - return missing; - } - - // collect field attributes used inside expressions - // TODO: Rather than going over all expressions manually, this should just call .references() - p.forEachExpression(TypedAttribute.class, f -> { + // Collect field attributes referenced by this plan but not yet present in the child's output. + // This is also correct for LookupJoinExec, where we only need field extraction on the left fields used to match, since the right + // side is always materialized. + p.references().forEach(f -> { if (f instanceof FieldAttribute || f instanceof MetadataAttribute) { if (input.contains(f) == false) { missing.add(f); } } }); + return missing; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 8c0488afdd42a..b85340936497e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -565,21 +565,12 @@ private PhysicalOperation planHashJoin(HashJoinExec join, LocalExecutionPlannerC private PhysicalOperation planLookupJoin(LookupJoinExec join, LocalExecutionPlannerContext context) { PhysicalOperation source = plan(join.left(), context); - // TODO: The source builder includes incoming fields including the ones we're going to drop Layout.Builder layoutBuilder = source.layout.builder(); for (Attribute f : join.addedFields()) { layoutBuilder.append(f); } Layout layout = layoutBuilder.build(); - // TODO: this works when the join happens on the coordinator - /* - * But when it happens on the data node we get a - * \_FieldExtractExec[language_code{f}#15, language_name{f}#16]<[]> - * \_EsQueryExec[languages_lookup], indexMode[lookup], query[][_doc{f}#18], limit[], sort[] estimatedRowSize[62] - * Which we'd prefer not to do - at least for now. We already know the fields we're loading - * and don't want any local planning. - */ EsQueryExec localSourceExec = (EsQueryExec) join.lookup(); if (localSourceExec.indexMode() != IndexMode.LOOKUP) { throw new IllegalArgumentException("can't plan [" + join + "]"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 71fba5683644d..4f7c620bc8d12 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -374,10 +374,11 @@ private void preAnalyzeLookupIndices(List indices, ListenerResult lis // call the EsqlResolveFieldsAction (field-caps) to resolve indices and get field types indexResolver.resolveAsMergedMapping( table.index(), - Set.of("*"), // Current LOOKUP JOIN syntax does not allow for field selection + Set.of("*"), // TODO: for LOOKUP JOIN, this currently declares all lookup index fields relevant and might fetch too many. null, listener.map(indexResolution -> listenerResult.withLookupIndexResolution(indexResolution)) ); + // TODO: Verify that the resolved index actually has indexMode: "lookup" } else { try { // No lookup indices specified diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index a63ee53cdd498..4e89a09db9ed4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -38,7 +38,7 @@ public static Analyzer defaultAnalyzer() { } public static Analyzer expandedDefaultAnalyzer() { - return analyzer(analyzerExpandedDefaultMapping()); + return analyzer(expandedDefaultIndexResolution()); } public static Analyzer analyzer(IndexResolution indexResolution) { @@ -47,18 +47,33 @@ public static Analyzer analyzer(IndexResolution indexResolution) { public static Analyzer analyzer(IndexResolution indexResolution, Verifier verifier) { return new Analyzer( - new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), indexResolution, defaultEnrichResolution()), + new AnalyzerContext( + EsqlTestUtils.TEST_CFG, + new EsqlFunctionRegistry(), + indexResolution, + defaultLookupResolution(), + defaultEnrichResolution() + ), verifier ); } public static Analyzer analyzer(IndexResolution indexResolution, Verifier verifier, Configuration config) { - return new Analyzer(new AnalyzerContext(config, new EsqlFunctionRegistry(), indexResolution, defaultEnrichResolution()), verifier); + return new Analyzer( + new AnalyzerContext(config, new EsqlFunctionRegistry(), indexResolution, defaultLookupResolution(), defaultEnrichResolution()), + verifier + ); } public static Analyzer analyzer(Verifier verifier) { return new Analyzer( - new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), analyzerDefaultMapping(), defaultEnrichResolution()), + new AnalyzerContext( + EsqlTestUtils.TEST_CFG, + new EsqlFunctionRegistry(), + analyzerDefaultMapping(), + defaultLookupResolution(), + defaultEnrichResolution() + ), verifier ); } @@ -98,10 +113,14 @@ public static IndexResolution analyzerDefaultMapping() { return loadMapping("mapping-basic.json", "test"); } - public static IndexResolution analyzerExpandedDefaultMapping() { + public static IndexResolution expandedDefaultIndexResolution() { return loadMapping("mapping-default.json", "test"); } + public static IndexResolution defaultLookupResolution() { + return loadMapping("mapping-languages.json", "languages_lookup"); + } + public static EnrichResolution defaultEnrichResolution() { EnrichResolution enrichResolution = new EnrichResolution(); loadEnrichPolicyResolution(enrichResolution, MATCH_TYPE, "languages", "language_code", "languages_idx", "mapping-languages.json"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 5a1e109041a16..6edbb55af463d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.LoadMapping; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; @@ -73,6 +74,8 @@ import static org.elasticsearch.xpack.esql.analysis.Analyzer.NO_FIELDS; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzer; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzerDefaultMapping; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.defaultEnrichResolution; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.tsdbIndexResolution; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; @@ -83,6 +86,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.matchesRegex; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.startsWith; //@TestLogging(value = "org.elasticsearch.xpack.esql.analysis:TRACE", reason = "debug") @@ -2002,6 +2006,58 @@ public void testLookupMatchTypeWrong() { assertThat(e.getMessage(), containsString("column type mismatch, table column was [integer] and original column was [keyword]")); } + public void testLookupJoinUnknownIndex() { + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V4.isEnabled()); + + String errorMessage = "Unknown index [foobar]"; + IndexResolution missingLookupIndex = IndexResolution.invalid(errorMessage); + + Analyzer analyzerMissingLookupIndex = new Analyzer( + new AnalyzerContext( + EsqlTestUtils.TEST_CFG, + new EsqlFunctionRegistry(), + analyzerDefaultMapping(), + missingLookupIndex, + defaultEnrichResolution() + ), + TEST_VERIFIER + ); + + String query = "FROM test | LOOKUP JOIN foobar ON last_name"; + + VerificationException e = expectThrows(VerificationException.class, () -> analyze(query, analyzerMissingLookupIndex)); + assertThat(e.getMessage(), containsString("1:25: " + errorMessage)); + + String query2 = "FROM test | LOOKUP JOIN foobar ON missing_field"; + + e = expectThrows(VerificationException.class, () -> analyze(query2, analyzerMissingLookupIndex)); + assertThat(e.getMessage(), containsString("1:25: " + errorMessage)); + assertThat(e.getMessage(), not(containsString("[missing_field]"))); + } + + public void testLookupJoinUnknownField() { + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V4.isEnabled()); + + String query = "FROM test | LOOKUP JOIN languages_lookup ON last_name"; + String errorMessage = "1:45: Unknown column [last_name] in right side of join"; + + VerificationException e = expectThrows(VerificationException.class, () -> analyze(query)); + assertThat(e.getMessage(), containsString(errorMessage)); + + String query2 = "FROM test | LOOKUP JOIN languages_lookup ON language_code"; + String errorMessage2 = "1:45: Unknown column [language_code] in left side of join"; + + e = expectThrows(VerificationException.class, () -> analyze(query2)); + assertThat(e.getMessage(), containsString(errorMessage2)); + + String query3 = "FROM test | LOOKUP JOIN languages_lookup ON missing_altogether"; + String errorMessage3 = "1:45: Unknown column [missing_altogether] in "; + + e = expectThrows(VerificationException.class, () -> analyze(query3)); + assertThat(e.getMessage(), containsString(errorMessage3 + "left side of join")); + assertThat(e.getMessage(), containsString(errorMessage3 + "right side of join")); + } + public void testImplicitCasting() { var e = expectThrows(VerificationException.class, () -> analyze(""" from test | eval x = concat("2024", "-04", "-01") + 1 day diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 74e2de1141728..882b8b7dbfd7c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1926,6 +1926,17 @@ public void testSortByAggregate() { assertEquals("1:18: Aggregate functions are not allowed in SORT [COUNT]", error("FROM test | SORT count(*)")); } + public void testLookupJoinDataTypeMismatch() { + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V4.isEnabled()); + + query("FROM test | EVAL language_code = languages | LOOKUP JOIN languages_lookup ON language_code"); + + assertEquals( + "1:87: JOIN left field [language_code] of type [KEYWORD] is incompatible with right field [language_code] of type [INTEGER]", + error("FROM test | EVAL language_code = languages::keyword | LOOKUP JOIN languages_lookup ON language_code") + ); + } + private void query(String query) { defaultAnalyzer.analyze(parser.createStatement(query)); } From 2fe6b60af14800a134db79d85b5f2dcb6df9b340 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 5 Dec 2024 12:22:22 +0100 Subject: [PATCH 042/204] Fix ProfileIntegTests (#117888) The test setup for `ProfileIntegTests` is flawed, where the full name of a user can be a substring of other profile names (e.g., `SER` is a substring of `User -space1`) -- when that's passed into suggest call with the `*` space, we get a match on all profiles, instead of only the one profile expected in the test, since we are matching on e.g. `SER*`. This PR restricts the setup to avoid the wildcard profile for that particular test. Closes: https://github.com/elastic/elasticsearch/issues/117782 --- .../xpack/security/profile/ProfileIntegTests.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java index 437fb76351176..3b55295c1efce 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java @@ -557,8 +557,11 @@ public void testSuggestProfilesWithHint() throws IOException { equalTo(profileHits4.subList(2, profileHits4.size())) ); + // Exclude profile for "*" space since that can match _all_ profiles, if the full name is a substring of "user" or the name of + // another profile + final List nonWildcardProfiles = profiles.stream().filter(p -> false == p.user().fullName().endsWith("*")).toList(); // A record will not be included if name does not match even when it has matching hint - final Profile hintedProfile5 = randomFrom(profiles); + final Profile hintedProfile5 = randomFrom(nonWildcardProfiles); final List profileHits5 = Arrays.stream( doSuggest( Set.of(), From 164d737b0556303d82fc84ce33bfa67ec8b7007b Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 5 Dec 2024 13:01:32 +0000 Subject: [PATCH 043/204] Speed up `testRespondAfterClose` (#117969) Since #106511 this test takes 30s because it waits for the client to time out and close the connection before allowing the transport to fully shut down. This commit reinstates the previous behaviour of closing connections quickly, and tests both client-triggered and server-triggered connection closure. --- .../Netty4HttpServerTransportTests.java | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java index 3fd5cc44a3403..1d39b993cef92 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java @@ -40,6 +40,7 @@ import io.netty.handler.codec.http.HttpUtil; import io.netty.handler.codec.http.HttpVersion; +import org.apache.http.ConnectionClosedException; import org.apache.http.HttpHost; import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchException; @@ -48,6 +49,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.IncrementalBulkService; import org.elasticsearch.action.support.ActionTestUtils; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClient; @@ -100,6 +102,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.concurrent.CancellationException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -110,6 +113,7 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN; import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_SERVER_SHUTDOWN_GRACE_PERIOD; import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; import static org.elasticsearch.rest.RestStatus.OK; import static org.elasticsearch.rest.RestStatus.UNAUTHORIZED; @@ -1039,8 +1043,16 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th } } - public void testRespondAfterClose() throws Exception { - final String url = "/thing"; + public void testRespondAfterServiceCloseWithClientCancel() throws Exception { + runRespondAfterServiceCloseTest(true); + } + + public void testRespondAfterServiceCloseWithServerCancel() throws Exception { + runRespondAfterServiceCloseTest(false); + } + + private void runRespondAfterServiceCloseTest(boolean clientCancel) throws Exception { + final String url = "/" + randomIdentifier(); final CountDownLatch responseReleasedLatch = new CountDownLatch(1); final SubscribableListener transportClosedFuture = new SubscribableListener<>(); final CountDownLatch handlingRequestLatch = new CountDownLatch(1); @@ -1066,7 +1078,9 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th try ( Netty4HttpServerTransport transport = new Netty4HttpServerTransport( - Settings.EMPTY, + clientCancel + ? Settings.EMPTY + : Settings.builder().put(SETTING_HTTP_SERVER_SHUTDOWN_GRACE_PERIOD.getKey(), TimeValue.timeValueMillis(1)).build(), networkService, threadPool, xContentRegistry(), @@ -1082,11 +1096,24 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th transport.start(); final var address = randomFrom(transport.boundAddress().boundAddresses()).address(); try (var client = RestClient.builder(new HttpHost(address.getAddress(), address.getPort())).build()) { - client.performRequestAsync(new Request("GET", url), ActionTestUtils.wrapAsRestResponseListener(ActionListener.noop())); + final var responseExceptionFuture = new PlainActionFuture(); + final var cancellable = client.performRequestAsync( + new Request("GET", url), + ActionTestUtils.wrapAsRestResponseListener(ActionTestUtils.assertNoSuccessListener(responseExceptionFuture::onResponse)) + ); safeAwait(handlingRequestLatch); + if (clientCancel) { + threadPool.generic().execute(cancellable::cancel); + } transport.close(); transportClosedFuture.onResponse(null); safeAwait(responseReleasedLatch); + final var responseException = safeGet(responseExceptionFuture); + if (clientCancel) { + assertThat(responseException, instanceOf(CancellationException.class)); + } else { + assertThat(responseException, instanceOf(ConnectionClosedException.class)); + } } } } From 5ff37d1cf7e026a08c0f6a5978e85619b527fe91 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 6 Dec 2024 00:12:50 +1100 Subject: [PATCH 044/204] Mute org.elasticsearch.packaging.test.ConfigurationTests test20HostnameSubstitution #118028 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index f525607a848d0..efd61da6c2dff 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -248,6 +248,9 @@ tests: - class: org.elasticsearch.index.reindex.ReindexNodeShutdownIT method: testReindexWithShutdown issue: https://github.com/elastic/elasticsearch/issues/118040 +- class: org.elasticsearch.packaging.test.ConfigurationTests + method: test20HostnameSubstitution + issue: https://github.com/elastic/elasticsearch/issues/118028 # Examples: # From f009147ce96368a5d52e076c518c92ac073892fe Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 6 Dec 2024 00:13:00 +1100 Subject: [PATCH 045/204] Mute org.elasticsearch.packaging.test.ArchiveTests test40AutoconfigurationNotTriggeredWhenNodeIsMeantToJoinExistingCluster #118029 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index efd61da6c2dff..b8d82f00bc43f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -251,6 +251,9 @@ tests: - class: org.elasticsearch.packaging.test.ConfigurationTests method: test20HostnameSubstitution issue: https://github.com/elastic/elasticsearch/issues/118028 +- class: org.elasticsearch.packaging.test.ArchiveTests + method: test40AutoconfigurationNotTriggeredWhenNodeIsMeantToJoinExistingCluster + issue: https://github.com/elastic/elasticsearch/issues/118029 # Examples: # From fdb1b2bf796fc26d85c98dc3e2c1913f76e35c8d Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Thu, 5 Dec 2024 14:20:31 +0000 Subject: [PATCH 046/204] Add a new `offset_source` field to store offsets referencing substrings of another field. (#118017) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This field is primarily designed for use with the `semantic_text` field, where it enables storing offsets that point to substrings of the field used to generate its underlying chunks. To prevent external usage, the field is intentionally undocumented, with detailed javadocs explaining its specific purpose and limitations. I couldn’t find a way to fully block external usage, but skipping the docs should keep it mostly out of sight for now. --- .../xpack/inference/InferencePlugin.java | 8 +- .../inference/mapper/OffsetSourceField.java | 145 ++++++++++ .../mapper/OffsetSourceFieldMapper.java | 253 ++++++++++++++++++ .../mapper/OffsetSourceFieldMapperTests.java | 216 +++++++++++++++ .../mapper/OffsetSourceFieldTests.java | 72 +++++ .../mapper/OffsetSourceFieldTypeTests.java | 44 +++ 6 files changed, 737 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceField.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapper.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldTypeTests.java diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index 48458bf4f5086..3c14e51a3c2d4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -68,6 +68,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.RequestExecutorServiceSettings; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.mapper.OffsetSourceFieldMapper; import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper; import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder; import org.elasticsearch.xpack.inference.rank.random.RandomRankBuilder; @@ -392,7 +393,12 @@ public void close() { @Override public Map getMappers() { - return Map.of(SemanticTextFieldMapper.CONTENT_TYPE, SemanticTextFieldMapper.PARSER); + return Map.of( + SemanticTextFieldMapper.CONTENT_TYPE, + SemanticTextFieldMapper.PARSER, + OffsetSourceFieldMapper.CONTENT_TYPE, + OffsetSourceFieldMapper.PARSER + ); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceField.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceField.java new file mode 100644 index 0000000000000..d8339f1004da2 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceField.java @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.mapper; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Represents a {@link Field} that stores a {@link Term} along with its start and end offsets. + * Note: The {@link Charset} used to calculate these offsets is not associated with this field. + * It is the responsibility of the consumer to handle the appropriate {@link Charset}. + */ +public final class OffsetSourceField extends Field { + private static final FieldType FIELD_TYPE = new FieldType(); + + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + } + + private int startOffset; + private int endOffset; + + public OffsetSourceField(String fieldName, String sourceFieldName, int startOffset, int endOffset) { + super(fieldName, sourceFieldName, FIELD_TYPE); + this.startOffset = startOffset; + this.endOffset = endOffset; + } + + public void setValues(String fieldName, int startOffset, int endOffset) { + this.fieldsData = fieldName; + this.startOffset = startOffset; + this.endOffset = endOffset; + } + + @Override + public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) { + OffsetTokenStream stream; + if (reuse instanceof OffsetTokenStream) { + stream = (OffsetTokenStream) reuse; + } else { + stream = new OffsetTokenStream(); + } + + stream.setValues((String) fieldsData, startOffset, endOffset); + return stream; + } + + public static OffsetSourceLoader loader(Terms terms) throws IOException { + return new OffsetSourceLoader(terms); + } + + private static final class OffsetTokenStream extends TokenStream { + private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class); + private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class); + private boolean used = true; + private String term = null; + private int startOffset = 0; + private int endOffset = 0; + + private OffsetTokenStream() {} + + /** Sets the values */ + void setValues(String term, int startOffset, int endOffset) { + this.term = term; + this.startOffset = startOffset; + this.endOffset = endOffset; + } + + @Override + public boolean incrementToken() { + if (used) { + return false; + } + clearAttributes(); + termAttribute.append(term); + offsetAttribute.setOffset(startOffset, endOffset); + used = true; + return true; + } + + @Override + public void reset() { + used = false; + } + + @Override + public void close() { + term = null; + } + } + + public static class OffsetSourceLoader { + private final Map postingsEnums = new LinkedHashMap<>(); + + private OffsetSourceLoader(Terms terms) throws IOException { + var termsEnum = terms.iterator(); + while (termsEnum.next() != null) { + var postings = termsEnum.postings(null, PostingsEnum.OFFSETS); + if (postings.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { + postingsEnums.put(termsEnum.term().utf8ToString(), postings); + } + } + } + + public OffsetSourceFieldMapper.OffsetSource advanceTo(int doc) throws IOException { + for (var it = postingsEnums.entrySet().iterator(); it.hasNext();) { + var entry = it.next(); + var postings = entry.getValue(); + if (postings.docID() < doc) { + if (postings.advance(doc) == DocIdSetIterator.NO_MORE_DOCS) { + it.remove(); + continue; + } + } + if (postings.docID() == doc) { + assert postings.freq() == 1; + postings.nextPosition(); + return new OffsetSourceFieldMapper.OffsetSource(entry.getKey(), postings.startOffset(), postings.endOffset()); + } + } + return null; + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapper.java new file mode 100644 index 0000000000000..e612076f1aaf2 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapper.java @@ -0,0 +1,253 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.mapper; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Query; +import org.elasticsearch.index.fielddata.FieldDataContext; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.DocumentParserContext; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; + +/** + * A {@link FieldMapper} that maps a field name to its start and end offsets. + * The {@link CharsetFormat} used to compute the offsets is specified via the charset parameter. + * Currently, only {@link CharsetFormat#UTF_16} is supported, aligning with Java's {@code String} charset + * for simpler internal usage and integration. + * + * Each document can store at most one value in this field. + * + * Note: This mapper is not yet documented and is intended exclusively for internal use by + * {@link SemanticTextFieldMapper}. If exposing this mapper directly to users becomes necessary, + * extending charset compatibility should be considered, as the current default (and sole supported charset) + * was chosen for ease of Java integration. + */ +public class OffsetSourceFieldMapper extends FieldMapper { + public static final String CONTENT_TYPE = "offset_source"; + + private static final String SOURCE_NAME_FIELD = "field"; + private static final String START_OFFSET_FIELD = "start"; + private static final String END_OFFSET_FIELD = "end"; + + public record OffsetSource(String field, int start, int end) implements ToXContentObject { + public OffsetSource { + if (start < 0 || end < 0) { + throw new IllegalArgumentException("Illegal offsets, expected positive numbers, got: " + start + ":" + end); + } + if (start > end) { + throw new IllegalArgumentException("Illegal offsets, expected start < end, got: " + start + " > " + end); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(SOURCE_NAME_FIELD, field); + builder.field(START_OFFSET_FIELD, start); + builder.field(END_OFFSET_FIELD, end); + return builder.endObject(); + } + } + + private static final ConstructingObjectParser OFFSET_SOURCE_PARSER = new ConstructingObjectParser<>( + CONTENT_TYPE, + true, + args -> new OffsetSource((String) args[0], (int) args[1], (int) args[2]) + ); + + static { + OFFSET_SOURCE_PARSER.declareString(constructorArg(), new ParseField(SOURCE_NAME_FIELD)); + OFFSET_SOURCE_PARSER.declareInt(constructorArg(), new ParseField(START_OFFSET_FIELD)); + OFFSET_SOURCE_PARSER.declareInt(constructorArg(), new ParseField(END_OFFSET_FIELD)); + } + + public enum CharsetFormat { + UTF_16(StandardCharsets.UTF_16); + + private Charset charSet; + + CharsetFormat(Charset charSet) { + this.charSet = charSet; + } + } + + public static class Builder extends FieldMapper.Builder { + private final Parameter charset = Parameter.enumParam( + "charset", + false, + i -> CharsetFormat.UTF_16, + CharsetFormat.UTF_16, + CharsetFormat.class + ); + private final Parameter> meta = Parameter.metaParam(); + + public Builder(String name) { + super(name); + } + + @Override + protected Parameter[] getParameters() { + return new Parameter[] { meta, charset }; + } + + @Override + public OffsetSourceFieldMapper build(MapperBuilderContext context) { + return new OffsetSourceFieldMapper( + leafName(), + new OffsetSourceFieldType(context.buildFullName(leafName()), charset.get(), meta.getValue()), + builderParams(this, context) + ); + } + } + + public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n)); + + public static final class OffsetSourceFieldType extends MappedFieldType { + private final CharsetFormat charset; + + public OffsetSourceFieldType(String name, CharsetFormat charset, Map meta) { + super(name, true, false, false, TextSearchInfo.NONE, meta); + this.charset = charset; + } + + public Charset getCharset() { + return charset.charSet; + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public boolean fieldHasValue(FieldInfos fieldInfos) { + return fieldInfos.fieldInfo(name()) != null; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { + throw new IllegalArgumentException("[offset_source] fields do not support sorting, scripting or aggregating"); + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + return new ValueFetcher() { + OffsetSourceField.OffsetSourceLoader offsetLoader; + + @Override + public void setNextReader(LeafReaderContext context) { + try { + var terms = context.reader().terms(name()); + offsetLoader = terms != null ? OffsetSourceField.loader(terms) : null; + } catch (IOException exc) { + throw new UncheckedIOException(exc); + } + } + + @Override + public List fetchValues(Source source, int doc, List ignoredValues) throws IOException { + var offsetSource = offsetLoader != null ? offsetLoader.advanceTo(doc) : null; + return offsetSource != null ? List.of(offsetSource) : null; + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + }; + } + + @Override + public Query termQuery(Object value, SearchExecutionContext context) { + throw new IllegalArgumentException("Queries on [offset_source] fields are not supported"); + } + + @Override + public boolean isSearchable() { + return false; + } + } + + /** + * @param simpleName the leaf name of the mapper + * @param mappedFieldType + * @param params initialization params for this field mapper + */ + protected OffsetSourceFieldMapper(String simpleName, MappedFieldType mappedFieldType, BuilderParams params) { + super(simpleName, mappedFieldType, params); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + protected boolean supportsParsingObject() { + return true; + } + + @Override + protected void parseCreateField(DocumentParserContext context) throws IOException { + var parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { + // skip + return; + } + + if (context.doc().getByKey(fullPath()) != null) { + throw new IllegalArgumentException( + "[offset_source] fields do not support indexing multiple values for the same field [" + + fullPath() + + "] in the same document" + ); + } + + // make sure that we don't expand dots in field names while parsing + boolean isWithinLeafObject = context.path().isWithinLeafObject(); + context.path().setWithinLeafObject(true); + try { + var offsetSource = OFFSET_SOURCE_PARSER.parse(parser, null); + context.doc() + .addWithKey( + fieldType().name(), + new OffsetSourceField(fullPath(), offsetSource.field, offsetSource.start, offsetSource.end) + ); + context.addToFieldNames(fieldType().name()); + } finally { + context.path().setWithinLeafObject(isWithinLeafObject); + } + } + + @Override + public FieldMapper.Builder getMergeBuilder() { + return new Builder(leafName()).init(this); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java new file mode 100644 index 0000000000000..40140d6da5eb5 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java @@ -0,0 +1,216 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.mapper; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.index.IndexableField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentParsingException; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperTestCase; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.search.lookup.SourceProvider; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.InferencePlugin; +import org.junit.AssumptionViolatedException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class OffsetSourceFieldMapperTests extends MapperTestCase { + @Override + protected Collection getPlugins() { + return List.of(new InferencePlugin(Settings.EMPTY)); + } + + @Override + protected void minimalMapping(XContentBuilder b) throws IOException { + b.field("type", "offset_source"); + } + + @Override + protected Object getSampleValueForDocument() { + return getSampleObjectForDocument(); + } + + @Override + protected Object getSampleObjectForDocument() { + return Map.of("field", "foo", "start", 100, "end", 300); + } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + return new OffsetSourceFieldMapper.OffsetSource("field", randomIntBetween(0, 100), randomIntBetween(101, 1000)); + } + + @Override + protected IngestScriptSupport ingestScriptSupport() { + throw new AssumptionViolatedException("not supported"); + } + + @Override + protected void registerParameters(ParameterChecker checker) throws IOException {} + + @Override + protected void assertSearchable(MappedFieldType fieldType) { + assertFalse(fieldType.isSearchable()); + } + + @Override + protected boolean supportsStoredFields() { + return false; + } + + @Override + protected boolean supportsEmptyInputArray() { + return false; + } + + @Override + protected boolean supportsCopyTo() { + return false; + } + + @Override + protected boolean supportsIgnoreMalformed() { + return false; + } + + @Override + protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { + return new SyntheticSourceSupport() { + @Override + public SyntheticSourceExample example(int maxValues) { + return new SyntheticSourceExample(getSampleValueForDocument(), getSampleValueForDocument(), null, b -> minimalMapping(b)); + } + + @Override + public List invalidExample() { + return List.of(); + } + }; + } + + @Override + public void testSyntheticSourceKeepArrays() { + // This mapper doesn't support multiple values (array of objects). + } + + public void testDefaults() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + assertEquals(Strings.toString(fieldMapping(this::minimalMapping)), mapper.mappingSource().toString()); + + ParsedDocument doc1 = mapper.parse( + source(b -> b.startObject("field").field("field", "foo").field("start", 0).field("end", 128).endObject()) + ); + List fields = doc1.rootDoc().getFields("field"); + assertEquals(1, fields.size()); + assertThat(fields.get(0), instanceOf(OffsetSourceField.class)); + OffsetSourceField offsetField1 = (OffsetSourceField) fields.get(0); + + ParsedDocument doc2 = mapper.parse( + source(b -> b.startObject("field").field("field", "bar").field("start", 128).field("end", 512).endObject()) + ); + OffsetSourceField offsetField2 = (OffsetSourceField) doc2.rootDoc().getFields("field").get(0); + + assertTokenStream(offsetField1.tokenStream(null, null), "foo", 0, 128); + assertTokenStream(offsetField2.tokenStream(null, null), "bar", 128, 512); + } + + private void assertTokenStream(TokenStream tk, String expectedTerm, int expectedStartOffset, int expectedEndOffset) throws IOException { + CharTermAttribute termAttribute = tk.addAttribute(CharTermAttribute.class); + OffsetAttribute offsetAttribute = tk.addAttribute(OffsetAttribute.class); + tk.reset(); + assertTrue(tk.incrementToken()); + assertThat(new String(termAttribute.buffer(), 0, termAttribute.length()), equalTo(expectedTerm)); + assertThat(offsetAttribute.startOffset(), equalTo(expectedStartOffset)); + assertThat(offsetAttribute.endOffset(), equalTo(expectedEndOffset)); + assertFalse(tk.incrementToken()); + } + + @Override + protected void assertFetch(MapperService mapperService, String field, Object value, String format) throws IOException { + MappedFieldType ft = mapperService.fieldType(field); + MappedFieldType.FielddataOperation fdt = MappedFieldType.FielddataOperation.SEARCH; + SourceToParse source = source(b -> b.field(ft.name(), value)); + SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.isSourceEnabled()).thenReturn(true); + when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); + when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> { + throw new UnsupportedOperationException(); + }, fdt)); + ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); + ParsedDocument doc = mapperService.documentMapper().parse(source); + withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { + Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); + nativeFetcher.setNextReader(ir.leaves().get(0)); + List fromNative = nativeFetcher.fetchValues(s, 0, new ArrayList<>()); + assertThat(fromNative.size(), equalTo(1)); + assertThat("fetching " + value, fromNative.get(0), equalTo(value)); + }); + } + + @Override + protected void assertFetchMany(MapperService mapperService, String field, Object value, String format, int count) throws IOException { + assumeFalse("[offset_source] currently don't support multiple values in the same field", false); + } + + public void testInvalidCharset() { + var exc = expectThrows(Exception.class, () -> createDocumentMapper(mapping(b -> { + b.startObject("field").field("type", "offset_source").field("charset", "utf_8").endObject(); + }))); + assertThat(exc.getCause().getMessage(), containsString("Unknown value [utf_8] for field [charset]")); + } + + public void testRejectMultiValuedFields() throws IOException { + DocumentMapper mapper = createDocumentMapper(mapping(b -> { b.startObject("field").field("type", "offset_source").endObject(); })); + + DocumentParsingException exc = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> { + b.startArray("field"); + { + b.startObject().field("field", "bar1").field("start", 128).field("end", 512).endObject(); + b.startObject().field("field", "bar2").field("start", 128).field("end", 512).endObject(); + } + b.endArray(); + }))); + assertThat(exc.getCause().getMessage(), containsString("[offset_source] fields do not support indexing multiple values")); + } + + public void testInvalidOffsets() throws IOException { + DocumentMapper mapper = createDocumentMapper(mapping(b -> { b.startObject("field").field("type", "offset_source").endObject(); })); + + DocumentParsingException exc = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> { + b.startArray("field"); + { + b.startObject().field("field", "bar1").field("start", -1).field("end", 512).endObject(); + } + b.endArray(); + }))); + assertThat(exc.getCause().getCause().getCause().getMessage(), containsString("Illegal offsets")); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldTests.java new file mode 100644 index 0000000000000..4d86263e446f8 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldTests.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.mapper; + +import org.apache.lucene.document.Document; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.test.ESTestCase; + +public class OffsetSourceFieldTests extends ESTestCase { + public void testBasics() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter( + random(), + dir, + newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean())) + ); + Document doc = new Document(); + OffsetSourceField field1 = new OffsetSourceField("field1", "foo", 1, 10); + doc.add(field1); + writer.addDocument(doc); + + field1.setValues("bar", 10, 128); + writer.addDocument(doc); + + writer.addDocument(new Document()); // gap + + field1.setValues("foo", 50, 256); + writer.addDocument(doc); + + writer.addDocument(new Document()); // double gap + writer.addDocument(new Document()); + + field1.setValues("baz", 32, 512); + writer.addDocument(doc); + + writer.forceMerge(1); + var reader = writer.getReader(); + writer.close(); + + var searcher = newSearcher(reader); + var context = searcher.getIndexReader().leaves().get(0); + + var terms = context.reader().terms("field1"); + assertNotNull(terms); + OffsetSourceField.OffsetSourceLoader loader = OffsetSourceField.loader(terms); + + var offset = loader.advanceTo(0); + assertEquals(new OffsetSourceFieldMapper.OffsetSource("foo", 1, 10), offset); + + offset = loader.advanceTo(1); + assertEquals(new OffsetSourceFieldMapper.OffsetSource("bar", 10, 128), offset); + + assertNull(loader.advanceTo(2)); + + offset = loader.advanceTo(3); + assertEquals(new OffsetSourceFieldMapper.OffsetSource("foo", 50, 256), offset); + + offset = loader.advanceTo(6); + assertEquals(new OffsetSourceFieldMapper.OffsetSource("baz", 32, 512), offset); + + assertNull(loader.advanceTo(189)); + + IOUtils.close(reader, dir); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldTypeTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldTypeTests.java new file mode 100644 index 0000000000000..ccb696515a060 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldTypeTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.mapper; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; + +import java.util.Collections; + +public class OffsetSourceFieldTypeTests extends FieldTypeTestCase { + public void testIsNotAggregatable() { + MappedFieldType fieldType = getMappedFieldType(); + assertFalse(fieldType.isAggregatable()); + } + + @Override + public void testFieldHasValue() { + MappedFieldType fieldType = getMappedFieldType(); + FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { getFieldInfoWithName(fieldType.name()) }); + assertTrue(fieldType.fieldHasValue(fieldInfos)); + } + + @Override + public void testFieldHasValueWithEmptyFieldInfos() { + MappedFieldType fieldType = getMappedFieldType(); + assertFalse(fieldType.fieldHasValue(FieldInfos.EMPTY)); + } + + @Override + public MappedFieldType getMappedFieldType() { + return new OffsetSourceFieldMapper.OffsetSourceFieldType( + "field", + OffsetSourceFieldMapper.CharsetFormat.UTF_16, + Collections.emptyMap() + ); + } +} From 422eb1af764f5f35f64016a8da038993b78ba69f Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 5 Dec 2024 15:25:15 +0100 Subject: [PATCH 047/204] Remove bucketOrd field from InternalTerms and friends (#118044) The field bucketOrd is only used for building the aggregation but has no use after that. --- .../search/aggregations/BucketOrder.java | 8 +- .../search/aggregations/InternalOrder.java | 21 +-- .../countedterms/CountedTermsAggregator.java | 89 ++++++----- .../bucket/terms/BucketPriorityQueue.java | 8 +- .../BucketSignificancePriorityQueue.java | 6 +- .../GlobalOrdinalsStringTermsAggregator.java | 144 +++++++++++------- .../terms/InternalSignificantTerms.java | 15 +- .../bucket/terms/InternalTerms.java | 10 -- .../terms/MapStringTermsAggregator.java | 103 +++++++------ .../bucket/terms/NumericTermsAggregator.java | 115 ++++++++------ .../bucket/terms/TermsAggregator.java | 6 +- .../bucket/terms/TermsAggregatorFactory.java | 6 +- .../multiterms/InternalMultiTerms.java | 3 - .../multiterms/MultiTermsAggregator.java | 103 +++++++------ 14 files changed, 355 insertions(+), 282 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/BucketOrder.java b/server/src/main/java/org/elasticsearch/search/aggregations/BucketOrder.java index 2d360705f75b6..c412ecb5d6361 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/BucketOrder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/BucketOrder.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket; +import org.elasticsearch.search.aggregations.bucket.terms.BucketAndOrd; import org.elasticsearch.search.aggregations.support.AggregationPath; import org.elasticsearch.xcontent.ToXContentObject; @@ -20,13 +21,12 @@ import java.util.Comparator; import java.util.List; import java.util.function.BiFunction; -import java.util.function.ToLongFunction; /** * {@link Bucket} ordering strategy. Buckets can be order either as * "complete" buckets using {@link #comparator()} or against a combination * of the buckets internals with its ordinal with - * {@link #partiallyBuiltBucketComparator(ToLongFunction, Aggregator)}. + * {@link #partiallyBuiltBucketComparator(Aggregator)}. */ public abstract class BucketOrder implements ToXContentObject, Writeable { /** @@ -102,7 +102,7 @@ public final void validate(Aggregator aggregator) throws AggregationExecutionExc * to validate this order because doing so checks all of the appropriate * paths. */ - partiallyBuiltBucketComparator(null, aggregator); + partiallyBuiltBucketComparator(aggregator); } /** @@ -121,7 +121,7 @@ public final void validate(Aggregator aggregator) throws AggregationExecutionExc * with it all the time. *

*/ - public abstract Comparator partiallyBuiltBucketComparator(ToLongFunction ordinalReader, Aggregator aggregator); + public abstract Comparator> partiallyBuiltBucketComparator(Aggregator aggregator); /** * Build a comparator for fully built buckets. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java index b2ca4a10dc4b3..3593eb5adf7e4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.search.aggregations.Aggregator.BucketComparator; import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket; +import org.elasticsearch.search.aggregations.bucket.terms.BucketAndOrd; import org.elasticsearch.search.aggregations.support.AggregationPath; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.sort.SortValue; @@ -30,7 +31,6 @@ import java.util.List; import java.util.Objects; import java.util.function.BiFunction; -import java.util.function.ToLongFunction; /** * Implementations for {@link Bucket} ordering strategies. @@ -63,10 +63,10 @@ public AggregationPath path() { } @Override - public Comparator partiallyBuiltBucketComparator(ToLongFunction ordinalReader, Aggregator aggregator) { + public Comparator> partiallyBuiltBucketComparator(Aggregator aggregator) { try { BucketComparator bucketComparator = path.bucketComparator(aggregator, order); - return (lhs, rhs) -> bucketComparator.compare(ordinalReader.applyAsLong(lhs), ordinalReader.applyAsLong(rhs)); + return (lhs, rhs) -> bucketComparator.compare(lhs.ord, rhs.ord); } catch (IllegalArgumentException e) { throw new AggregationExecutionException.InvalidPath("Invalid aggregation order path [" + path + "]. " + e.getMessage(), e); } @@ -188,12 +188,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public Comparator partiallyBuiltBucketComparator(ToLongFunction ordinalReader, Aggregator aggregator) { - List> comparators = orderElements.stream() - .map(oe -> oe.partiallyBuiltBucketComparator(ordinalReader, aggregator)) - .toList(); + public Comparator> partiallyBuiltBucketComparator(Aggregator aggregator) { + List>> comparators = new ArrayList<>(orderElements.size()); + for (BucketOrder order : orderElements) { + comparators.add(order.partiallyBuiltBucketComparator(aggregator)); + } return (lhs, rhs) -> { - for (Comparator c : comparators) { + for (Comparator> c : comparators) { int result = c.compare(lhs, rhs); if (result != 0) { return result; @@ -299,9 +300,9 @@ byte id() { } @Override - public Comparator partiallyBuiltBucketComparator(ToLongFunction ordinalReader, Aggregator aggregator) { + public Comparator> partiallyBuiltBucketComparator(Aggregator aggregator) { Comparator comparator = comparator(); - return comparator::compare; + return (lhs, rhs) -> comparator.compare(lhs.bucket, rhs.bucket); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregator.java index 344b90b06c4f6..571ce3a9a4519 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregator.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.IntArray; import org.elasticsearch.common.util.LongArray; import org.elasticsearch.common.util.ObjectArray; import org.elasticsearch.core.Releasables; @@ -26,6 +27,7 @@ import org.elasticsearch.search.aggregations.InternalOrder; import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.bucket.terms.BucketAndOrd; import org.elasticsearch.search.aggregations.bucket.terms.BucketPriorityQueue; import org.elasticsearch.search.aggregations.bucket.terms.BytesKeyedBucketOrds; import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms; @@ -38,7 +40,6 @@ import java.util.Arrays; import java.util.Map; import java.util.function.BiConsumer; -import java.util.function.Supplier; import static java.util.Collections.emptyList; import static org.elasticsearch.search.aggregations.InternalOrder.isKeyOrder; @@ -115,51 +116,57 @@ public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throw LongArray otherDocCounts = bigArrays().newLongArray(owningBucketOrds.size()); ObjectArray topBucketsPerOrd = bigArrays().newObjectArray(owningBucketOrds.size()) ) { - for (long ordIdx = 0; ordIdx < topBucketsPerOrd.size(); ordIdx++) { - int size = (int) Math.min(bucketOrds.size(), bucketCountThresholds.getShardSize()); - - // as users can't control sort order, in practice we'll always sort by doc count descending - try ( - BucketPriorityQueue ordered = new BucketPriorityQueue<>( - size, - bigArrays(), - partiallyBuiltBucketComparator - ) - ) { - StringTerms.Bucket spare = null; - BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrds.get(ordIdx)); - Supplier emptyBucketBuilder = () -> new StringTerms.Bucket( - new BytesRef(), - 0, - null, - false, - 0, - format - ); - while (ordsEnum.next()) { - long docCount = bucketDocCount(ordsEnum.ord()); - otherDocCounts.increment(ordIdx, docCount); - if (spare == null) { - checkRealMemoryCBForInternalBucket(); - spare = emptyBucketBuilder.get(); + try (IntArray bucketsToCollect = bigArrays().newIntArray(owningBucketOrds.size())) { + // find how many buckets we are going to collect + long ordsToCollect = 0; + for (long ordIdx = 0; ordIdx < owningBucketOrds.size(); ordIdx++) { + int size = (int) Math.min(bucketOrds.bucketsInOrd(owningBucketOrds.get(ordIdx)), bucketCountThresholds.getShardSize()); + bucketsToCollect.set(ordIdx, size); + ordsToCollect += size; + } + try (LongArray ordsArray = bigArrays().newLongArray(ordsToCollect)) { + long ordsCollected = 0; + for (long ordIdx = 0; ordIdx < owningBucketOrds.size(); ordIdx++) { + // as users can't control sort order, in practice we'll always sort by doc count descending + try ( + BucketPriorityQueue ordered = new BucketPriorityQueue<>( + bucketsToCollect.get(ordIdx), + bigArrays(), + order.partiallyBuiltBucketComparator(this) + ) + ) { + BucketAndOrd spare = null; + BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrds.get(ordIdx)); + while (ordsEnum.next()) { + long docCount = bucketDocCount(ordsEnum.ord()); + otherDocCounts.increment(ordIdx, docCount); + if (spare == null) { + checkRealMemoryCBForInternalBucket(); + spare = new BucketAndOrd<>(new StringTerms.Bucket(new BytesRef(), 0, null, false, 0, format)); + } + ordsEnum.readValue(spare.bucket.getTermBytes()); + spare.bucket.setDocCount(docCount); + spare.ord = ordsEnum.ord(); + spare = ordered.insertWithOverflow(spare); + } + final int orderedSize = (int) ordered.size(); + final StringTerms.Bucket[] buckets = new StringTerms.Bucket[orderedSize]; + for (int i = orderedSize - 1; i >= 0; --i) { + BucketAndOrd bucketAndOrd = ordered.pop(); + buckets[i] = bucketAndOrd.bucket; + ordsArray.set(ordsCollected + i, bucketAndOrd.ord); + otherDocCounts.increment(ordIdx, -bucketAndOrd.bucket.getDocCount()); + bucketAndOrd.bucket.setTermBytes(BytesRef.deepCopyOf(bucketAndOrd.bucket.getTermBytes())); + } + topBucketsPerOrd.set(ordIdx, buckets); + ordsCollected += orderedSize; } - ordsEnum.readValue(spare.getTermBytes()); - spare.setDocCount(docCount); - spare.setBucketOrd(ordsEnum.ord()); - spare = ordered.insertWithOverflow(spare); - } - - topBucketsPerOrd.set(ordIdx, new StringTerms.Bucket[(int) ordered.size()]); - for (int i = (int) ordered.size() - 1; i >= 0; --i) { - topBucketsPerOrd.get(ordIdx)[i] = ordered.pop(); - otherDocCounts.increment(ordIdx, -topBucketsPerOrd.get(ordIdx)[i].getDocCount()); - topBucketsPerOrd.get(ordIdx)[i].setTermBytes(BytesRef.deepCopyOf(topBucketsPerOrd.get(ordIdx)[i].getTermBytes())); } + assert ordsCollected == ordsArray.size(); + buildSubAggsForAllBuckets(topBucketsPerOrd, ordsArray, InternalTerms.Bucket::setAggregations); } } - buildSubAggsForAllBuckets(topBucketsPerOrd, InternalTerms.Bucket::getBucketOrd, InternalTerms.Bucket::setAggregations); - return buildAggregations(Math.toIntExact(owningBucketOrds.size()), ordIdx -> { final BucketOrder reduceOrder; if (isKeyOrder(order) == false) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/BucketPriorityQueue.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/BucketPriorityQueue.java index 7f8e5c8c885fa..9550003a5bd1e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/BucketPriorityQueue.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/BucketPriorityQueue.java @@ -13,17 +13,17 @@ import java.util.Comparator; -public class BucketPriorityQueue extends ObjectArrayPriorityQueue { +public class BucketPriorityQueue extends ObjectArrayPriorityQueue> { - private final Comparator comparator; + private final Comparator> comparator; - public BucketPriorityQueue(int size, BigArrays bigArrays, Comparator comparator) { + public BucketPriorityQueue(int size, BigArrays bigArrays, Comparator> comparator) { super(size, bigArrays); this.comparator = comparator; } @Override - protected boolean lessThan(B a, B b) { + protected boolean lessThan(BucketAndOrd a, BucketAndOrd b) { return comparator.compare(a, b) > 0; // reverse, since we reverse again when adding to a list } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/BucketSignificancePriorityQueue.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/BucketSignificancePriorityQueue.java index fe751c9e79189..4736f52d93622 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/BucketSignificancePriorityQueue.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/BucketSignificancePriorityQueue.java @@ -12,14 +12,14 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ObjectArrayPriorityQueue; -public class BucketSignificancePriorityQueue extends ObjectArrayPriorityQueue { +public class BucketSignificancePriorityQueue extends ObjectArrayPriorityQueue> { public BucketSignificancePriorityQueue(int size, BigArrays bigArrays) { super(size, bigArrays); } @Override - protected boolean lessThan(SignificantTerms.Bucket o1, SignificantTerms.Bucket o2) { - return o1.getSignificanceScore() < o2.getSignificanceScore(); + protected boolean lessThan(BucketAndOrd o1, BucketAndOrd o2) { + return o1.bucket.getSignificanceScore() < o2.bucket.getSignificanceScore(); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java index 0ec03a6f56dd9..439b61cc43ddf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java @@ -20,6 +20,7 @@ import org.apache.lucene.util.PriorityQueue; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.IntArray; import org.elasticsearch.common.util.LongArray; import org.elasticsearch.common.util.LongHash; import org.elasticsearch.common.util.ObjectArray; @@ -561,10 +562,10 @@ InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOExc ) { GlobalOrdLookupFunction lookupGlobalOrd = valuesSupplier.get()::lookupOrd; final int size = (int) Math.min(valueCount, bucketCountThresholds.getShardSize()); - try (ObjectArrayPriorityQueue ordered = collectionStrategy.buildPriorityQueue(size)) { + try (ObjectArrayPriorityQueue> ordered = collectionStrategy.buildPriorityQueue(size)) { BucketUpdater updater = collectionStrategy.bucketUpdater(0, lookupGlobalOrd); collect(new BucketInfoConsumer() { - TB spare = null; + BucketAndOrd spare = null; @Override public void accept(long globalOrd, long bucketOrd, long docCount) throws IOException { @@ -572,24 +573,31 @@ public void accept(long globalOrd, long bucketOrd, long docCount) throws IOExcep if (docCount >= bucketCountThresholds.getShardMinDocCount()) { if (spare == null) { checkRealMemoryCBForInternalBucket(); - spare = collectionStrategy.buildEmptyTemporaryBucket(); + spare = new BucketAndOrd<>(collectionStrategy.buildEmptyTemporaryBucket()); } - updater.updateBucket(spare, globalOrd, bucketOrd, docCount); + spare.ord = bucketOrd; + updater.updateBucket(spare.bucket, globalOrd, docCount); spare = ordered.insertWithOverflow(spare); } } }); // Get the top buckets - topBucketsPreOrd.set(0, collectionStrategy.buildBuckets((int) ordered.size())); - for (int i = (int) ordered.size() - 1; i >= 0; --i) { - checkRealMemoryCBForInternalBucket(); - B bucket = collectionStrategy.convertTempBucketToRealBucket(ordered.pop(), lookupGlobalOrd); - topBucketsPreOrd.get(0)[i] = bucket; - otherDocCount.increment(0, -bucket.getDocCount()); + int orderedSize = (int) ordered.size(); + try (LongArray ordsArray = bigArrays().newLongArray(orderedSize)) { + B[] buckets = collectionStrategy.buildBuckets(orderedSize); + for (int i = orderedSize - 1; i >= 0; --i) { + checkRealMemoryCBForInternalBucket(); + BucketAndOrd bucketAndOrd = ordered.pop(); + B bucket = collectionStrategy.convertTempBucketToRealBucket(bucketAndOrd.bucket, lookupGlobalOrd); + ordsArray.set(i, bucketAndOrd.ord); + buckets[i] = bucket; + otherDocCount.increment(0, -bucket.getDocCount()); + } + topBucketsPreOrd.set(0, buckets); + collectionStrategy.buildSubAggs(topBucketsPreOrd, ordsArray); } } - collectionStrategy.buildSubAggs(topBucketsPreOrd); return GlobalOrdinalsStringTermsAggregator.this.buildAggregations( Math.toIntExact(owningBucketOrds.size()), ordIdx -> collectionStrategy.buildResult( @@ -710,39 +718,61 @@ InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOExc LongArray otherDocCount = bigArrays().newLongArray(owningBucketOrds.size(), true); ObjectArray topBucketsPreOrd = collectionStrategy.buildTopBucketsPerOrd(owningBucketOrds.size()) ) { - GlobalOrdLookupFunction lookupGlobalOrd = valuesSupplier.get()::lookupOrd; - for (long ordIdx = 0; ordIdx < topBucketsPreOrd.size(); ordIdx++) { - long owningBucketOrd = owningBucketOrds.get(ordIdx); - collectZeroDocEntriesIfNeeded(owningBucketOrds.get(ordIdx)); - int size = (int) Math.min(bucketOrds.bucketsInOrd(owningBucketOrd), bucketCountThresholds.getShardSize()); - try (ObjectArrayPriorityQueue ordered = collectionStrategy.buildPriorityQueue(size)) { - BucketUpdater updater = collectionStrategy.bucketUpdater(owningBucketOrd, lookupGlobalOrd); - LongKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrd); - TB spare = null; - while (ordsEnum.next()) { - long docCount = bucketDocCount(ordsEnum.ord()); - otherDocCount.increment(ordIdx, docCount); - if (docCount < bucketCountThresholds.getShardMinDocCount()) { - continue; - } - if (spare == null) { - checkRealMemoryCBForInternalBucket(); - spare = collectionStrategy.buildEmptyTemporaryBucket(); + try (IntArray bucketsToCollect = bigArrays().newIntArray(owningBucketOrds.size())) { + long ordsToCollect = 0; + for (long ordIdx = 0; ordIdx < owningBucketOrds.size(); ordIdx++) { + final long owningBucketOrd = owningBucketOrds.get(ordIdx); + collectZeroDocEntriesIfNeeded(owningBucketOrd); + final int size = (int) Math.min(bucketOrds.bucketsInOrd(owningBucketOrd), bucketCountThresholds.getShardSize()); + ordsToCollect += size; + bucketsToCollect.set(ordIdx, size); + } + try (LongArray ordsArray = bigArrays().newLongArray(ordsToCollect)) { + long ordsCollected = 0; + GlobalOrdLookupFunction lookupGlobalOrd = valuesSupplier.get()::lookupOrd; + for (long ordIdx = 0; ordIdx < topBucketsPreOrd.size(); ordIdx++) { + long owningBucketOrd = owningBucketOrds.get(ordIdx); + try ( + ObjectArrayPriorityQueue> ordered = collectionStrategy.buildPriorityQueue( + bucketsToCollect.get(ordIdx) + ) + ) { + BucketUpdater updater = collectionStrategy.bucketUpdater(owningBucketOrd, lookupGlobalOrd); + LongKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrd); + BucketAndOrd spare = null; + while (ordsEnum.next()) { + long docCount = bucketDocCount(ordsEnum.ord()); + otherDocCount.increment(ordIdx, docCount); + if (docCount < bucketCountThresholds.getShardMinDocCount()) { + continue; + } + if (spare == null) { + checkRealMemoryCBForInternalBucket(); + spare = new BucketAndOrd<>(collectionStrategy.buildEmptyTemporaryBucket()); + } + updater.updateBucket(spare.bucket, ordsEnum.value(), docCount); + spare.ord = ordsEnum.ord(); + spare = ordered.insertWithOverflow(spare); + } + // Get the top buckets + int orderedSize = (int) ordered.size(); + B[] buckets = collectionStrategy.buildBuckets(orderedSize); + for (int i = orderedSize - 1; i >= 0; --i) { + checkRealMemoryCBForInternalBucket(); + BucketAndOrd bucketAndOrd = ordered.pop(); + B bucket = collectionStrategy.convertTempBucketToRealBucket(bucketAndOrd.bucket, lookupGlobalOrd); + ordsArray.set(ordsCollected + i, bucketAndOrd.ord); + buckets[i] = bucket; + otherDocCount.increment(ordIdx, -bucket.getDocCount()); + } + topBucketsPreOrd.set(ordIdx, buckets); + ordsCollected += orderedSize; } - updater.updateBucket(spare, ordsEnum.value(), ordsEnum.ord(), docCount); - spare = ordered.insertWithOverflow(spare); - } - // Get the top buckets - topBucketsPreOrd.set(ordIdx, collectionStrategy.buildBuckets((int) ordered.size())); - for (int i = (int) ordered.size() - 1; i >= 0; --i) { - checkRealMemoryCBForInternalBucket(); - B bucket = collectionStrategy.convertTempBucketToRealBucket(ordered.pop(), lookupGlobalOrd); - topBucketsPreOrd.get(ordIdx)[i] = bucket; - otherDocCount.increment(ordIdx, -bucket.getDocCount()); } + assert ordsCollected == ordsArray.size(); + collectionStrategy.buildSubAggs(topBucketsPreOrd, ordsArray); } } - collectionStrategy.buildSubAggs(topBucketsPreOrd); return GlobalOrdinalsStringTermsAggregator.this.buildAggregations( Math.toIntExact(owningBucketOrds.size()), ordIdx -> collectionStrategy.buildResult( @@ -791,7 +821,7 @@ abstract class ResultStrategy< * Build a {@link PriorityQueue} to sort the buckets. After we've * collected all of the buckets we'll collect all entries in the queue. */ - abstract ObjectArrayPriorityQueue buildPriorityQueue(int size); + abstract ObjectArrayPriorityQueue> buildPriorityQueue(int size); /** * Build an array to hold the "top" buckets for each ordinal. @@ -813,7 +843,7 @@ abstract class ResultStrategy< * Build the sub-aggregations into the buckets. This will usually * delegate to {@link #buildSubAggsForAllBuckets}. */ - abstract void buildSubAggs(ObjectArray topBucketsPreOrd) throws IOException; + abstract void buildSubAggs(ObjectArray topBucketsPreOrd, LongArray ordsArray) throws IOException; /** * Turn the buckets into an aggregation result. @@ -834,7 +864,7 @@ abstract class ResultStrategy< } interface BucketUpdater { - void updateBucket(TB spare, long globalOrd, long bucketOrd, long docCount) throws IOException; + void updateBucket(TB spare, long globalOrd, long docCount) throws IOException; } /** @@ -868,29 +898,30 @@ OrdBucket buildEmptyTemporaryBucket() { @Override BucketUpdater bucketUpdater(long owningBucketOrd, GlobalOrdLookupFunction lookupGlobalOrd) { - return (spare, globalOrd, bucketOrd, docCount) -> { + return (spare, globalOrd, docCount) -> { spare.globalOrd = globalOrd; - spare.bucketOrd = bucketOrd; spare.docCount = docCount; }; } @Override - ObjectArrayPriorityQueue buildPriorityQueue(int size) { - return new BucketPriorityQueue<>(size, bigArrays(), partiallyBuiltBucketComparator); + ObjectArrayPriorityQueue> buildPriorityQueue(int size) { + return new BucketPriorityQueue<>( + size, + bigArrays(), + order.partiallyBuiltBucketComparator(GlobalOrdinalsStringTermsAggregator.this) + ); } @Override StringTerms.Bucket convertTempBucketToRealBucket(OrdBucket temp, GlobalOrdLookupFunction lookupGlobalOrd) throws IOException { BytesRef term = BytesRef.deepCopyOf(lookupGlobalOrd.apply(temp.globalOrd)); - StringTerms.Bucket result = new StringTerms.Bucket(term, temp.docCount, null, showTermDocCountError, 0, format); - result.bucketOrd = temp.bucketOrd; - return result; + return new StringTerms.Bucket(term, temp.docCount, null, showTermDocCountError, 0, format); } @Override - void buildSubAggs(ObjectArray topBucketsPreOrd) throws IOException { - buildSubAggsForAllBuckets(topBucketsPreOrd, b -> b.bucketOrd, (b, aggs) -> b.aggregations = aggs); + void buildSubAggs(ObjectArray topBucketsPreOrd, LongArray ordsArray) throws IOException { + buildSubAggsForAllBuckets(topBucketsPreOrd, ordsArray, (b, aggs) -> b.aggregations = aggs); } @Override @@ -1005,8 +1036,7 @@ private long subsetSize(long owningBucketOrd) { @Override BucketUpdater bucketUpdater(long owningBucketOrd, GlobalOrdLookupFunction lookupGlobalOrd) { long subsetSize = subsetSize(owningBucketOrd); - return (spare, globalOrd, bucketOrd, docCount) -> { - spare.bucketOrd = bucketOrd; + return (spare, globalOrd, docCount) -> { oversizedCopy(lookupGlobalOrd.apply(globalOrd), spare.termBytes); spare.subsetDf = docCount; spare.supersetDf = backgroundFrequencies.freq(spare.termBytes); @@ -1020,7 +1050,7 @@ BucketUpdater bucketUpdater(long owningBucketOrd, } @Override - ObjectArrayPriorityQueue buildPriorityQueue(int size) { + ObjectArrayPriorityQueue> buildPriorityQueue(int size) { return new BucketSignificancePriorityQueue<>(size, bigArrays()); } @@ -1033,8 +1063,8 @@ SignificantStringTerms.Bucket convertTempBucketToRealBucket( } @Override - void buildSubAggs(ObjectArray topBucketsPreOrd) throws IOException { - buildSubAggsForAllBuckets(topBucketsPreOrd, b -> b.bucketOrd, (b, aggs) -> b.aggregations = aggs); + void buildSubAggs(ObjectArray topBucketsPreOrd, LongArray ordsArray) throws IOException { + buildSubAggsForAllBuckets(topBucketsPreOrd, ordsArray, (b, aggs) -> b.aggregations = aggs); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalSignificantTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalSignificantTerms.java index 78ae2481f5d99..5108793b8a809 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalSignificantTerms.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalSignificantTerms.java @@ -10,12 +10,12 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.ObjectArrayPriorityQueue; import org.elasticsearch.common.util.ObjectObjectPagedHashMap; import org.elasticsearch.core.Releasables; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationErrors; import org.elasticsearch.search.aggregations.AggregationReduceContext; -import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorReducer; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; @@ -58,12 +58,6 @@ public interface Reader> { long subsetDf; long supersetDf; - /** - * Ordinal of the bucket while it is being built. Not used after it is - * returned from {@link Aggregator#buildAggregations(org.elasticsearch.common.util.LongArray)} and not - * serialized. - */ - transient long bucketOrd; double score; protected InternalAggregations aggregations; final transient DocValueFormat format; @@ -235,7 +229,12 @@ canLeadReduction here is essentially checking if this shard returned data. Unma public InternalAggregation get() { final SignificanceHeuristic heuristic = getSignificanceHeuristic().rewrite(reduceContext); final int size = (int) (reduceContext.isFinalReduce() == false ? buckets.size() : Math.min(requiredSize, buckets.size())); - try (BucketSignificancePriorityQueue ordered = new BucketSignificancePriorityQueue<>(size, reduceContext.bigArrays())) { + try (ObjectArrayPriorityQueue ordered = new ObjectArrayPriorityQueue(size, reduceContext.bigArrays()) { + @Override + protected boolean lessThan(B a, B b) { + return a.getSignificanceScore() < b.getSignificanceScore(); + } + }) { buckets.forEach(entry -> { final B b = createBucket( entry.value.subsetDf[0], diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java index 739f0b923eaab..de35046691b34 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java @@ -38,8 +38,6 @@ public interface Reader> { B read(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException; } - long bucketOrd; - protected long docCount; private long docCountError; protected InternalAggregations aggregations; @@ -88,14 +86,6 @@ public void setDocCount(long docCount) { this.docCount = docCount; } - public long getBucketOrd() { - return bucketOrd; - } - - public void setBucketOrd(long bucketOrd) { - this.bucketOrd = bucketOrd; - } - @Override public long getDocCountError() { return docCountError; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/MapStringTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/MapStringTermsAggregator.java index b96c495d37489..026912a583ef3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/MapStringTermsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/MapStringTermsAggregator.java @@ -17,6 +17,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.PriorityQueue; +import org.elasticsearch.common.util.IntArray; import org.elasticsearch.common.util.LongArray; import org.elasticsearch.common.util.ObjectArray; import org.elasticsearch.common.util.ObjectArrayPriorityQueue; @@ -43,6 +44,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Comparator; import java.util.Map; import java.util.function.BiConsumer; import java.util.function.Function; @@ -287,40 +289,55 @@ private InternalAggregation[] buildAggregations(LongArray owningBucketOrds) thro LongArray otherDocCounts = bigArrays().newLongArray(owningBucketOrds.size(), true); ObjectArray topBucketsPerOrd = buildTopBucketsPerOrd(Math.toIntExact(owningBucketOrds.size())) ) { - for (long ordIdx = 0; ordIdx < topBucketsPerOrd.size(); ordIdx++) { - long owningOrd = owningBucketOrds.get(ordIdx); - collectZeroDocEntriesIfNeeded(owningOrd, excludeDeletedDocs); - int size = (int) Math.min(bucketOrds.size(), bucketCountThresholds.getShardSize()); - - try (ObjectArrayPriorityQueue ordered = buildPriorityQueue(size)) { - B spare = null; - BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningOrd); - BucketUpdater bucketUpdater = bucketUpdater(owningOrd); - while (ordsEnum.next()) { - long docCount = bucketDocCount(ordsEnum.ord()); - otherDocCounts.increment(ordIdx, docCount); - if (docCount < bucketCountThresholds.getShardMinDocCount()) { - continue; - } - if (spare == null) { - checkRealMemoryCBForInternalBucket(); - spare = buildEmptyBucket(); + try (IntArray bucketsToCollect = bigArrays().newIntArray(owningBucketOrds.size())) { + long ordsToCollect = 0; + for (long ordIdx = 0; ordIdx < owningBucketOrds.size(); ordIdx++) { + final long owningBucketOrd = owningBucketOrds.get(ordIdx); + collectZeroDocEntriesIfNeeded(owningBucketOrd, excludeDeletedDocs); + final int size = (int) Math.min(bucketOrds.bucketsInOrd(owningBucketOrd), bucketCountThresholds.getShardSize()); + ordsToCollect += size; + bucketsToCollect.set(ordIdx, size); + } + try (LongArray ordsArray = bigArrays().newLongArray(ordsToCollect)) { + long ordsCollected = 0; + for (long ordIdx = 0; ordIdx < topBucketsPerOrd.size(); ordIdx++) { + long owningOrd = owningBucketOrds.get(ordIdx); + try (ObjectArrayPriorityQueue> ordered = buildPriorityQueue(bucketsToCollect.get(ordIdx))) { + BucketAndOrd spare = null; + BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningOrd); + BucketUpdater bucketUpdater = bucketUpdater(owningOrd); + while (ordsEnum.next()) { + long docCount = bucketDocCount(ordsEnum.ord()); + otherDocCounts.increment(ordIdx, docCount); + if (docCount < bucketCountThresholds.getShardMinDocCount()) { + continue; + } + if (spare == null) { + checkRealMemoryCBForInternalBucket(); + spare = new BucketAndOrd<>(buildEmptyBucket()); + } + bucketUpdater.updateBucket(spare.bucket, ordsEnum, docCount); + spare.ord = ordsEnum.ord(); + spare = ordered.insertWithOverflow(spare); + } + + final int orderedSize = (int) ordered.size(); + final B[] buckets = buildBuckets(orderedSize); + for (int i = orderedSize - 1; i >= 0; --i) { + BucketAndOrd bucketAndOrd = ordered.pop(); + finalizeBucket(bucketAndOrd.bucket); + buckets[i] = bucketAndOrd.bucket; + ordsArray.set(ordsCollected + i, bucketAndOrd.ord); + otherDocCounts.increment(ordIdx, -bucketAndOrd.bucket.getDocCount()); + } + topBucketsPerOrd.set(ordIdx, buckets); + ordsCollected += orderedSize; } - bucketUpdater.updateBucket(spare, ordsEnum, docCount); - spare = ordered.insertWithOverflow(spare); - } - - topBucketsPerOrd.set(ordIdx, buildBuckets((int) ordered.size())); - for (int i = (int) ordered.size() - 1; i >= 0; --i) { - topBucketsPerOrd.get(ordIdx)[i] = ordered.pop(); - otherDocCounts.increment(ordIdx, -topBucketsPerOrd.get(ordIdx)[i].getDocCount()); - finalizeBucket(topBucketsPerOrd.get(ordIdx)[i]); } + assert ordsCollected == ordsArray.size(); + buildSubAggs(topBucketsPerOrd, ordsArray); } } - - buildSubAggs(topBucketsPerOrd); - return MapStringTermsAggregator.this.buildAggregations( Math.toIntExact(owningBucketOrds.size()), ordIdx -> buildResult(owningBucketOrds.get(ordIdx), otherDocCounts.get(ordIdx), topBucketsPerOrd.get(ordIdx)) @@ -355,7 +372,7 @@ private InternalAggregation[] buildAggregations(LongArray owningBucketOrds) thro * Build a {@link PriorityQueue} to sort the buckets. After we've * collected all of the buckets we'll collect all entries in the queue. */ - abstract ObjectArrayPriorityQueue buildPriorityQueue(int size); + abstract ObjectArrayPriorityQueue> buildPriorityQueue(int size); /** * Update fields in {@code spare} to reflect information collected for @@ -382,9 +399,9 @@ private InternalAggregation[] buildAggregations(LongArray owningBucketOrds) thro /** * Build the sub-aggregations into the buckets. This will usually - * delegate to {@link #buildSubAggsForAllBuckets}. + * delegate to {@link #buildSubAggsForAllBuckets(ObjectArray, LongArray, BiConsumer)}. */ - abstract void buildSubAggs(ObjectArray topBucketsPerOrd) throws IOException; + abstract void buildSubAggs(ObjectArray topBucketsPerOrd, LongArray ordsArray) throws IOException; /** * Turn the buckets into an aggregation result. @@ -407,9 +424,11 @@ interface BucketUpdater */ class StandardTermsResults extends ResultStrategy { private final ValuesSource valuesSource; + private final Comparator> comparator; - StandardTermsResults(ValuesSource valuesSource) { + StandardTermsResults(ValuesSource valuesSource, Aggregator aggregator) { this.valuesSource = valuesSource; + this.comparator = order.partiallyBuiltBucketComparator(aggregator); } @Override @@ -498,8 +517,8 @@ StringTerms.Bucket buildEmptyBucket() { } @Override - ObjectArrayPriorityQueue buildPriorityQueue(int size) { - return new BucketPriorityQueue<>(size, bigArrays(), partiallyBuiltBucketComparator); + ObjectArrayPriorityQueue> buildPriorityQueue(int size) { + return new BucketPriorityQueue<>(size, bigArrays(), comparator); } @Override @@ -507,7 +526,6 @@ BucketUpdater bucketUpdater(long owningBucketOrd) { return (spare, ordsEnum, docCount) -> { ordsEnum.readValue(spare.termBytes); spare.docCount = docCount; - spare.bucketOrd = ordsEnum.ord(); }; } @@ -532,8 +550,8 @@ void finalizeBucket(StringTerms.Bucket bucket) { } @Override - void buildSubAggs(ObjectArray topBucketsPerOrd) throws IOException { - buildSubAggsForAllBuckets(topBucketsPerOrd, b -> b.bucketOrd, (b, a) -> b.aggregations = a); + void buildSubAggs(ObjectArray topBucketsPerOrd, LongArray ordArray) throws IOException { + buildSubAggsForAllBuckets(topBucketsPerOrd, ordArray, (b, a) -> b.aggregations = a); } @Override @@ -625,7 +643,7 @@ SignificantStringTerms.Bucket buildEmptyBucket() { } @Override - ObjectArrayPriorityQueue buildPriorityQueue(int size) { + ObjectArrayPriorityQueue> buildPriorityQueue(int size) { return new BucketSignificancePriorityQueue<>(size, bigArrays()); } @@ -634,7 +652,6 @@ BucketUpdater bucketUpdater(long owningBucketOrd) long subsetSize = subsetSizes.get(owningBucketOrd); return (spare, ordsEnum, docCount) -> { ordsEnum.readValue(spare.termBytes); - spare.bucketOrd = ordsEnum.ord(); spare.subsetDf = docCount; spare.supersetDf = backgroundFrequencies.freq(spare.termBytes); /* @@ -667,8 +684,8 @@ void finalizeBucket(SignificantStringTerms.Bucket bucket) { } @Override - void buildSubAggs(ObjectArray topBucketsPerOrd) throws IOException { - buildSubAggsForAllBuckets(topBucketsPerOrd, b -> b.bucketOrd, (b, a) -> b.aggregations = a); + void buildSubAggs(ObjectArray topBucketsPerOrd, LongArray ordsArray) throws IOException { + buildSubAggsForAllBuckets(topBucketsPerOrd, ordsArray, (b, a) -> b.aggregations = a); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregator.java index 5d4c15d8a3b80..a54053f712f8d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/NumericTermsAggregator.java @@ -14,6 +14,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.util.IntArray; import org.elasticsearch.common.util.LongArray; import org.elasticsearch.common.util.ObjectArray; import org.elasticsearch.common.util.ObjectArrayPriorityQueue; @@ -40,6 +41,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Comparator; import java.util.Map; import java.util.function.BiConsumer; import java.util.function.Function; @@ -167,42 +169,56 @@ private InternalAggregation[] buildAggregations(LongArray owningBucketOrds) thro LongArray otherDocCounts = bigArrays().newLongArray(owningBucketOrds.size(), true); ObjectArray topBucketsPerOrd = buildTopBucketsPerOrd(owningBucketOrds.size()) ) { - for (long ordIdx = 0; ordIdx < topBucketsPerOrd.size(); ordIdx++) { - final long owningBucketOrd = owningBucketOrds.get(ordIdx); - collectZeroDocEntriesIfNeeded(owningBucketOrd, excludeDeletedDocs); - long bucketsInOrd = bucketOrds.bucketsInOrd(owningBucketOrd); - - int size = (int) Math.min(bucketsInOrd, bucketCountThresholds.getShardSize()); - try (ObjectArrayPriorityQueue ordered = buildPriorityQueue(size)) { - B spare = null; - BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrd); - BucketUpdater bucketUpdater = bucketUpdater(owningBucketOrd); - while (ordsEnum.next()) { - long docCount = bucketDocCount(ordsEnum.ord()); - otherDocCounts.increment(ordIdx, docCount); - if (docCount < bucketCountThresholds.getShardMinDocCount()) { - continue; - } - if (spare == null) { - checkRealMemoryCBForInternalBucket(); - spare = buildEmptyBucket(); - } - bucketUpdater.updateBucket(spare, ordsEnum, docCount); - spare = ordered.insertWithOverflow(spare); - } + try (IntArray bucketsToCollect = bigArrays().newIntArray(owningBucketOrds.size())) { + long ordsToCollect = 0; + for (long ordIdx = 0; ordIdx < owningBucketOrds.size(); ordIdx++) { + final long owningBucketOrd = owningBucketOrds.get(ordIdx); + collectZeroDocEntriesIfNeeded(owningBucketOrd, excludeDeletedDocs); + int size = (int) Math.min(bucketOrds.bucketsInOrd(owningBucketOrd), bucketCountThresholds.getShardSize()); + bucketsToCollect.set(ordIdx, size); + ordsToCollect += size; + } + try (LongArray ordsArray = bigArrays().newLongArray(ordsToCollect)) { + long ordsCollected = 0; + for (long ordIdx = 0; ordIdx < topBucketsPerOrd.size(); ordIdx++) { + final long owningBucketOrd = owningBucketOrds.get(ordIdx); + try (ObjectArrayPriorityQueue> ordered = buildPriorityQueue(bucketsToCollect.get(ordIdx))) { + BucketAndOrd spare = null; + BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrd); + BucketUpdater bucketUpdater = bucketUpdater(owningBucketOrd); + while (ordsEnum.next()) { + long docCount = bucketDocCount(ordsEnum.ord()); + otherDocCounts.increment(ordIdx, docCount); + if (docCount < bucketCountThresholds.getShardMinDocCount()) { + continue; + } + if (spare == null) { + checkRealMemoryCBForInternalBucket(); + spare = new BucketAndOrd<>(buildEmptyBucket()); + } + bucketUpdater.updateBucket(spare.bucket, ordsEnum, docCount); + spare.ord = ordsEnum.ord(); + spare = ordered.insertWithOverflow(spare); + } + + // Get the top buckets + final int orderedSize = (int) ordered.size(); + final B[] bucketsForOrd = buildBuckets(orderedSize); + for (int b = orderedSize - 1; b >= 0; --b) { + BucketAndOrd bucketAndOrd = ordered.pop(); + bucketsForOrd[b] = bucketAndOrd.bucket; + ordsArray.set(ordsCollected + b, bucketAndOrd.ord); + otherDocCounts.increment(ordIdx, -bucketAndOrd.bucket.getDocCount()); + } + topBucketsPerOrd.set(ordIdx, bucketsForOrd); + ordsCollected += orderedSize; - // Get the top buckets - B[] bucketsForOrd = buildBuckets((int) ordered.size()); - topBucketsPerOrd.set(ordIdx, bucketsForOrd); - for (int b = (int) ordered.size() - 1; b >= 0; --b) { - topBucketsPerOrd.get(ordIdx)[b] = ordered.pop(); - otherDocCounts.increment(ordIdx, -topBucketsPerOrd.get(ordIdx)[b].getDocCount()); + } } + assert ordsCollected == ordsArray.size(); + buildSubAggs(topBucketsPerOrd, ordsArray); } } - - buildSubAggs(topBucketsPerOrd); - return NumericTermsAggregator.this.buildAggregations( Math.toIntExact(owningBucketOrds.size()), ordIdx -> buildResult(owningBucketOrds.get(ordIdx), otherDocCounts.get(ordIdx), topBucketsPerOrd.get(ordIdx)) @@ -254,13 +270,13 @@ private InternalAggregation[] buildAggregations(LongArray owningBucketOrds) thro * Build a {@link ObjectArrayPriorityQueue} to sort the buckets. After we've * collected all of the buckets we'll collect all entries in the queue. */ - abstract ObjectArrayPriorityQueue buildPriorityQueue(int size); + abstract ObjectArrayPriorityQueue> buildPriorityQueue(int size); /** * Build the sub-aggregations into the buckets. This will usually - * delegate to {@link #buildSubAggsForAllBuckets}. + * delegate to {@link #buildSubAggsForAllBuckets(ObjectArray, LongArray, BiConsumer)}. */ - abstract void buildSubAggs(ObjectArray topBucketsPerOrd) throws IOException; + abstract void buildSubAggs(ObjectArray topBucketsPerOrd, LongArray ordsArray) throws IOException; /** * Collect extra entries for "zero" hit documents if they were requested @@ -287,9 +303,11 @@ interface BucketUpdater abstract class StandardTermsResultStrategy, B extends InternalTerms.Bucket> extends ResultStrategy { protected final boolean showTermDocCountError; + private final Comparator> comparator; - StandardTermsResultStrategy(boolean showTermDocCountError) { + StandardTermsResultStrategy(boolean showTermDocCountError, Aggregator aggregator) { this.showTermDocCountError = showTermDocCountError; + this.comparator = order.partiallyBuiltBucketComparator(aggregator); } @Override @@ -298,13 +316,13 @@ final LeafBucketCollector wrapCollector(LeafBucketCollector primary) { } @Override - final ObjectArrayPriorityQueue buildPriorityQueue(int size) { - return new BucketPriorityQueue<>(size, bigArrays(), partiallyBuiltBucketComparator); + final ObjectArrayPriorityQueue> buildPriorityQueue(int size) { + return new BucketPriorityQueue<>(size, bigArrays(), comparator); } @Override - final void buildSubAggs(ObjectArray topBucketsPerOrd) throws IOException { - buildSubAggsForAllBuckets(topBucketsPerOrd, b -> b.bucketOrd, (b, aggs) -> b.aggregations = aggs); + final void buildSubAggs(ObjectArray topBucketsPerOrd, LongArray ordsArray) throws IOException { + buildSubAggsForAllBuckets(topBucketsPerOrd, ordsArray, (b, aggs) -> b.aggregations = aggs); } @Override @@ -340,8 +358,8 @@ public final void close() {} } class LongTermsResults extends StandardTermsResultStrategy { - LongTermsResults(boolean showTermDocCountError) { - super(showTermDocCountError); + LongTermsResults(boolean showTermDocCountError, Aggregator aggregator) { + super(showTermDocCountError, aggregator); } @Override @@ -374,7 +392,6 @@ BucketUpdater bucketUpdater(long owningBucketOrd) { return (LongTerms.Bucket spare, BucketOrdsEnum ordsEnum, long docCount) -> { spare.term = ordsEnum.value(); spare.docCount = docCount; - spare.bucketOrd = ordsEnum.ord(); }; } @@ -424,8 +441,8 @@ LongTerms buildEmptyResult() { class DoubleTermsResults extends StandardTermsResultStrategy { - DoubleTermsResults(boolean showTermDocCountError) { - super(showTermDocCountError); + DoubleTermsResults(boolean showTermDocCountError, Aggregator aggregator) { + super(showTermDocCountError, aggregator); } @Override @@ -458,7 +475,6 @@ BucketUpdater bucketUpdater(long owningBucketOrd) { return (DoubleTerms.Bucket spare, BucketOrdsEnum ordsEnum, long docCount) -> { spare.term = NumericUtils.sortableLongToDouble(ordsEnum.value()); spare.docCount = docCount; - spare.bucketOrd = ordsEnum.ord(); }; } @@ -575,7 +591,6 @@ BucketUpdater bucketUpdater(long owningBucketOrd) { spare.term = ordsEnum.value(); spare.subsetDf = docCount; spare.supersetDf = backgroundFrequencies.freq(spare.term); - spare.bucketOrd = ordsEnum.ord(); // During shard-local down-selection we use subset/superset stats that are for this shard only // Back at the central reducer these properties will be updated with global stats spare.updateScore(significanceHeuristic, subsetSize, supersetSize); @@ -583,13 +598,13 @@ BucketUpdater bucketUpdater(long owningBucketOrd) { } @Override - ObjectArrayPriorityQueue buildPriorityQueue(int size) { + ObjectArrayPriorityQueue> buildPriorityQueue(int size) { return new BucketSignificancePriorityQueue<>(size, bigArrays()); } @Override - void buildSubAggs(ObjectArray topBucketsPerOrd) throws IOException { - buildSubAggsForAllBuckets(topBucketsPerOrd, b -> b.bucketOrd, (b, aggs) -> b.aggregations = aggs); + void buildSubAggs(ObjectArray topBucketsPerOrd, LongArray ordsArray) throws IOException { + buildSubAggsForAllBuckets(topBucketsPerOrd, ordsArray, (b, aggs) -> b.aggregations = aggs); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java index 4922be7cec1ba..c07c0726a4ae1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java @@ -27,7 +27,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.Comparator; import java.util.HashSet; import java.util.Map; import java.util.Objects; @@ -190,7 +189,6 @@ public boolean equals(Object obj) { protected final DocValueFormat format; protected final BucketCountThresholds bucketCountThresholds; protected final BucketOrder order; - protected final Comparator> partiallyBuiltBucketComparator; protected final Set aggsUsedForSorting; protected final SubAggCollectionMode collectMode; @@ -209,7 +207,9 @@ public TermsAggregator( super(name, factories, context, parent, metadata); this.bucketCountThresholds = bucketCountThresholds; this.order = order; - partiallyBuiltBucketComparator = order == null ? null : order.partiallyBuiltBucketComparator(b -> b.bucketOrd, this); + if (order != null) { + order.validate(this); + } this.format = format; if ((subAggsNeedScore() && descendsFromNestedAggregator(parent)) || context.isInSortOrderExecutionRequired()) { /** diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java index 2c7b768fcdbb3..da5ae37b08228 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java @@ -195,12 +195,12 @@ private static TermsAggregatorSupplier numericSupplier() { if (includeExclude != null) { longFilter = includeExclude.convertToDoubleFilter(); } - resultStrategy = agg -> agg.new DoubleTermsResults(showTermDocCountError); + resultStrategy = agg -> agg.new DoubleTermsResults(showTermDocCountError, agg); } else { if (includeExclude != null) { longFilter = includeExclude.convertToLongFilter(valuesSourceConfig.format()); } - resultStrategy = agg -> agg.new LongTermsResults(showTermDocCountError); + resultStrategy = agg -> agg.new LongTermsResults(showTermDocCountError, agg); } return new NumericTermsAggregator( name, @@ -403,7 +403,7 @@ Aggregator create( name, factories, new MapStringTermsAggregator.ValuesSourceCollectorSource(valuesSourceConfig), - a -> a.new StandardTermsResults(valuesSourceConfig.getValuesSource()), + a -> a.new StandardTermsResults(valuesSourceConfig.getValuesSource(), a), order, valuesSourceConfig.format(), bucketCountThresholds, diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/InternalMultiTerms.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/InternalMultiTerms.java index 0d42a2856a10e..85510c8a989c0 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/InternalMultiTerms.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/InternalMultiTerms.java @@ -37,9 +37,6 @@ public class InternalMultiTerms extends AbstractInternalTerms { - - long bucketOrd; - protected long docCount; protected InternalAggregations aggregations; private long docCountError; diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregator.java index 1691aedf543f4..5c10e2c8feeb1 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregator.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.IntArray; import org.elasticsearch.common.util.LongArray; import org.elasticsearch.common.util.ObjectArray; import org.elasticsearch.common.util.ObjectArrayPriorityQueue; @@ -40,6 +41,7 @@ import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; import org.elasticsearch.search.aggregations.bucket.DeferableBucketAggregator; +import org.elasticsearch.search.aggregations.bucket.terms.BucketAndOrd; import org.elasticsearch.search.aggregations.bucket.terms.BucketPriorityQueue; import org.elasticsearch.search.aggregations.bucket.terms.BytesKeyedBucketOrds; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; @@ -72,7 +74,7 @@ class MultiTermsAggregator extends DeferableBucketAggregator { protected final List formats; protected final TermsAggregator.BucketCountThresholds bucketCountThresholds; protected final BucketOrder order; - protected final Comparator partiallyBuiltBucketComparator; + protected final Comparator> partiallyBuiltBucketComparator; protected final Set aggsUsedForSorting; protected final SubAggCollectionMode collectMode; private final List values; @@ -99,7 +101,7 @@ protected MultiTermsAggregator( super(name, factories, context, parent, metadata); this.bucketCountThresholds = bucketCountThresholds; this.order = order; - partiallyBuiltBucketComparator = order == null ? null : order.partiallyBuiltBucketComparator(b -> b.bucketOrd, this); + partiallyBuiltBucketComparator = order == null ? null : order.partiallyBuiltBucketComparator(this); this.formats = formats; this.showTermDocCountError = showTermDocCountError; if (subAggsNeedScore() && descendsFromNestedAggregator(parent) || context.isInSortOrderExecutionRequired()) { @@ -242,52 +244,67 @@ public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throw LongArray otherDocCounts = bigArrays().newLongArray(owningBucketOrds.size(), true); ObjectArray topBucketsPerOrd = bigArrays().newObjectArray(owningBucketOrds.size()) ) { - for (long ordIdx = 0; ordIdx < owningBucketOrds.size(); ordIdx++) { - final long owningBucketOrd = owningBucketOrds.get(ordIdx); - long bucketsInOrd = bucketOrds.bucketsInOrd(owningBucketOrd); - - int size = (int) Math.min(bucketsInOrd, bucketCountThresholds.getShardSize()); - try ( - ObjectArrayPriorityQueue ordered = new BucketPriorityQueue<>( - size, - bigArrays(), - partiallyBuiltBucketComparator - ) - ) { - InternalMultiTerms.Bucket spare = null; - BytesRef spareKey = null; - BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrd); - while (ordsEnum.next()) { - long docCount = bucketDocCount(ordsEnum.ord()); - otherDocCounts.increment(ordIdx, docCount); - if (docCount < bucketCountThresholds.getShardMinDocCount()) { - continue; - } - if (spare == null) { - checkRealMemoryCBForInternalBucket(); - spare = new InternalMultiTerms.Bucket(null, 0, null, showTermDocCountError, 0, formats, keyConverters); - spareKey = new BytesRef(); - } - ordsEnum.readValue(spareKey); - spare.terms = unpackTerms(spareKey); - spare.docCount = docCount; - spare.bucketOrd = ordsEnum.ord(); - spare = ordered.insertWithOverflow(spare); - } + try (IntArray bucketsToCollect = bigArrays().newIntArray(owningBucketOrds.size())) { + long ordsToCollect = 0; + for (long ordIdx = 0; ordIdx < owningBucketOrds.size(); ordIdx++) { + int size = (int) Math.min(bucketOrds.bucketsInOrd(owningBucketOrds.get(ordIdx)), bucketCountThresholds.getShardSize()); + ordsToCollect += size; + bucketsToCollect.set(ordIdx, size); + } + try (LongArray ordsArray = bigArrays().newLongArray(ordsToCollect)) { + long ordsCollected = 0; + for (long ordIdx = 0; ordIdx < owningBucketOrds.size(); ordIdx++) { + final long owningBucketOrd = owningBucketOrds.get(ordIdx); + long bucketsInOrd = bucketOrds.bucketsInOrd(owningBucketOrd); + + int size = (int) Math.min(bucketsInOrd, bucketCountThresholds.getShardSize()); + try ( + ObjectArrayPriorityQueue> ordered = new BucketPriorityQueue<>( + size, + bigArrays(), + partiallyBuiltBucketComparator + ) + ) { + BucketAndOrd spare = null; + BytesRef spareKey = null; + BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrd); + while (ordsEnum.next()) { + long docCount = bucketDocCount(ordsEnum.ord()); + otherDocCounts.increment(ordIdx, docCount); + if (docCount < bucketCountThresholds.getShardMinDocCount()) { + continue; + } + if (spare == null) { + checkRealMemoryCBForInternalBucket(); + spare = new BucketAndOrd<>( + new InternalMultiTerms.Bucket(null, 0, null, showTermDocCountError, 0, formats, keyConverters) + ); + spareKey = new BytesRef(); + } + ordsEnum.readValue(spareKey); + spare.bucket.terms = unpackTerms(spareKey); + spare.bucket.docCount = docCount; + spare.ord = ordsEnum.ord(); + spare = ordered.insertWithOverflow(spare); + } - // Get the top buckets - InternalMultiTerms.Bucket[] bucketsForOrd = new InternalMultiTerms.Bucket[(int) ordered.size()]; - topBucketsPerOrd.set(ordIdx, bucketsForOrd); - for (int b = (int) ordered.size() - 1; b >= 0; --b) { - InternalMultiTerms.Bucket[] buckets = topBucketsPerOrd.get(ordIdx); - buckets[b] = ordered.pop(); - otherDocCounts.increment(ordIdx, -buckets[b].getDocCount()); + // Get the top buckets + int orderedSize = (int) ordered.size(); + InternalMultiTerms.Bucket[] buckets = new InternalMultiTerms.Bucket[orderedSize]; + for (int i = orderedSize - 1; i >= 0; --i) { + BucketAndOrd bucketAndOrd = ordered.pop(); + buckets[i] = bucketAndOrd.bucket; + ordsArray.set(ordsCollected + i, bucketAndOrd.ord); + otherDocCounts.increment(ordIdx, -buckets[i].getDocCount()); + } + topBucketsPerOrd.set(ordIdx, buckets); + ordsCollected += orderedSize; + } } + buildSubAggsForAllBuckets(topBucketsPerOrd, ordsArray, (b, a) -> b.aggregations = a); } } - buildSubAggsForAllBuckets(topBucketsPerOrd, b -> b.bucketOrd, (b, a) -> b.aggregations = a); - return buildAggregations( Math.toIntExact(owningBucketOrds.size()), ordIdx -> buildResult(otherDocCounts.get(ordIdx), topBucketsPerOrd.get(ordIdx)) From 1fecab19254715941f42bdebe025298e89d5574b Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 5 Dec 2024 16:25:32 +0100 Subject: [PATCH 048/204] Update synthetic source cutoff date (#118069) Updating from 01-02-2025T00:00:00UTC to 04-02-2025T00:00:00UTC --- .../xpack/logsdb/SyntheticSourceLicenseService.java | 2 +- .../xpack/logsdb/LegacyLicenceIntegrationTests.java | 3 ++- ...SyntheticSourceIndexSettingsProviderLegacyLicenseTests.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java index 26a672fb1c903..e629f9b3998bb 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java @@ -29,7 +29,7 @@ final class SyntheticSourceLicenseService { // You can only override this property if you received explicit approval from Elastic. static final String CUTOFF_DATE_SYS_PROP_NAME = "es.mapping.synthetic_source_fallback_to_stored_source.cutoff_date_restricted_override"; private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceLicenseService.class); - static final long DEFAULT_CUTOFF_DATE = LocalDateTime.of(2025, 2, 1, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); + static final long DEFAULT_CUTOFF_DATE = LocalDateTime.of(2025, 2, 4, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); /** * A setting that determines whether source mode should always be stored source. Regardless of licence. diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java index 890bc464a2579..f8f307b572f33 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java @@ -69,7 +69,8 @@ public void testSyntheticSourceUsageWithLegacyLicense() { } public void testSyntheticSourceUsageWithLegacyLicensePastCutoff() throws Exception { - long startPastCutoff = LocalDateTime.of(2025, 11, 12, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); + // One day after default cutoff date + long startPastCutoff = LocalDateTime.of(2025, 2, 5, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); putLicense(createGoldOrPlatinumLicense(startPastCutoff)); ensureGreen(); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderLegacyLicenseTests.java index eda0d87868745..c871a7d0216ed 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderLegacyLicenseTests.java @@ -98,7 +98,7 @@ public void testGetAdditionalIndexSettingsTsdb() throws IOException { } public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception { - long start = LocalDateTime.of(2025, 2, 2, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); + long start = LocalDateTime.of(2025, 2, 5, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); License license = createGoldOrPlatinumLicense(start); long time = LocalDateTime.of(2024, 12, 31, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); var licenseState = new XPackLicenseState(() -> time, new XPackLicenseStatus(license.operationMode(), true, null)); From 9d350537608e89624b660ff40f8b96275d8ba9d9 Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:32:59 +0100 Subject: [PATCH 049/204] Adds warning to Create inference API page (#118073) --- docs/reference/inference/put-inference.asciidoc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index ed93c290b6ad4..4f82889f562d8 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -10,7 +10,6 @@ Creates an {infer} endpoint to perform an {infer} task. * For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. ==== - [discrete] [[put-inference-api-request]] ==== {api-request-title} @@ -47,6 +46,14 @@ Refer to the service list in the <> API. In the response, look for `"state": "fully_allocated"` and ensure the `"allocation_count"` matches the `"target_allocation_count"`. +* Avoid creating multiple endpoints for the same model unless required, as each endpoint consumes significant resources. +==== + + The following services are available through the {infer} API. You can find the available task types next to the service name. Click the links to review the configuration details of the services: From 5d1bca34f9dbfd3904c624a0f48a474e557577e5 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 5 Dec 2024 17:22:03 +0100 Subject: [PATCH 050/204] Make NestedHelper a utility class (#118071) Noticed instantiating these instances taking a visible and unexpected amount of CPU in profiles (probably from bootstrapping the lambda/callsite for the predicate). This fixes the logic to effectively disappear from profiling and makes it easier to reason about as well by removing the indirect use of the search context and just explicitly passing it around. No need to instantiate instances of this thing either, escape analysis probably isn't able to remove it because of the recursive instance method calls. --- .../index/query/NestedQueryBuilder.java | 3 +- .../index/search/NestedHelper.java | 59 ++-- .../search/DefaultSearchContext.java | 5 +- .../search/vectors/KnnVectorQueryBuilder.java | 3 +- .../index/search/NestedHelperTests.java | 274 ++++++++++-------- .../authz/permission/DocumentPermissions.java | 6 +- .../planner/EsPhysicalOperationProviders.java | 14 +- 7 files changed, 185 insertions(+), 179 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java index 83bca7d27aeeb..503b2adf756f5 100644 --- a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java @@ -321,8 +321,7 @@ public static Query toQuery( // ToParentBlockJoinQuery requires that the inner query only matches documents // in its child space - NestedHelper nestedHelper = new NestedHelper(context.nestedLookup(), context::isFieldMapped); - if (nestedHelper.mightMatchNonNestedDocs(innerQuery, path)) { + if (NestedHelper.mightMatchNonNestedDocs(innerQuery, path, context)) { innerQuery = Queries.filtered(innerQuery, mapper.nestedTypeFilter()); } diff --git a/server/src/main/java/org/elasticsearch/index/search/NestedHelper.java b/server/src/main/java/org/elasticsearch/index/search/NestedHelper.java index 96e8ac35c8e32..a04f930e052b9 100644 --- a/server/src/main/java/org/elasticsearch/index/search/NestedHelper.java +++ b/server/src/main/java/org/elasticsearch/index/search/NestedHelper.java @@ -21,29 +21,21 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.NestedObjectMapper; - -import java.util.function.Predicate; +import org.elasticsearch.index.query.SearchExecutionContext; /** Utility class to filter parent and children clauses when building nested * queries. */ public final class NestedHelper { - private final NestedLookup nestedLookup; - private final Predicate isMappedFieldPredicate; - - public NestedHelper(NestedLookup nestedLookup, Predicate isMappedFieldPredicate) { - this.nestedLookup = nestedLookup; - this.isMappedFieldPredicate = isMappedFieldPredicate; - } + private NestedHelper() {} /** Returns true if the given query might match nested documents. */ - public boolean mightMatchNestedDocs(Query query) { + public static boolean mightMatchNestedDocs(Query query, SearchExecutionContext searchExecutionContext) { if (query instanceof ConstantScoreQuery) { - return mightMatchNestedDocs(((ConstantScoreQuery) query).getQuery()); + return mightMatchNestedDocs(((ConstantScoreQuery) query).getQuery(), searchExecutionContext); } else if (query instanceof BoostQuery) { - return mightMatchNestedDocs(((BoostQuery) query).getQuery()); + return mightMatchNestedDocs(((BoostQuery) query).getQuery(), searchExecutionContext); } else if (query instanceof MatchAllDocsQuery) { return true; } else if (query instanceof MatchNoDocsQuery) { @@ -51,17 +43,17 @@ public boolean mightMatchNestedDocs(Query query) { } else if (query instanceof TermQuery) { // We only handle term(s) queries and range queries, which should already // cover a high majority of use-cases - return mightMatchNestedDocs(((TermQuery) query).getTerm().field()); + return mightMatchNestedDocs(((TermQuery) query).getTerm().field(), searchExecutionContext); } else if (query instanceof TermInSetQuery tis) { if (tis.getTermsCount() > 0) { - return mightMatchNestedDocs(tis.getField()); + return mightMatchNestedDocs(tis.getField(), searchExecutionContext); } else { return false; } } else if (query instanceof PointRangeQuery) { - return mightMatchNestedDocs(((PointRangeQuery) query).getField()); + return mightMatchNestedDocs(((PointRangeQuery) query).getField(), searchExecutionContext); } else if (query instanceof IndexOrDocValuesQuery) { - return mightMatchNestedDocs(((IndexOrDocValuesQuery) query).getIndexQuery()); + return mightMatchNestedDocs(((IndexOrDocValuesQuery) query).getIndexQuery(), searchExecutionContext); } else if (query instanceof final BooleanQuery bq) { final boolean hasRequiredClauses = bq.clauses().stream().anyMatch(BooleanClause::isRequired); if (hasRequiredClauses) { @@ -69,13 +61,13 @@ public boolean mightMatchNestedDocs(Query query) { .stream() .filter(BooleanClause::isRequired) .map(BooleanClause::query) - .allMatch(this::mightMatchNestedDocs); + .allMatch(f -> mightMatchNestedDocs(f, searchExecutionContext)); } else { return bq.clauses() .stream() .filter(c -> c.occur() == Occur.SHOULD) .map(BooleanClause::query) - .anyMatch(this::mightMatchNestedDocs); + .anyMatch(f -> mightMatchNestedDocs(f, searchExecutionContext)); } } else if (query instanceof ESToParentBlockJoinQuery) { return ((ESToParentBlockJoinQuery) query).getPath() != null; @@ -85,7 +77,7 @@ public boolean mightMatchNestedDocs(Query query) { } /** Returns true if a query on the given field might match nested documents. */ - boolean mightMatchNestedDocs(String field) { + private static boolean mightMatchNestedDocs(String field, SearchExecutionContext searchExecutionContext) { if (field.startsWith("_")) { // meta field. Every meta field behaves differently, eg. nested // documents have the same _uid as their parent, put their path in @@ -94,36 +86,36 @@ boolean mightMatchNestedDocs(String field) { // we might add a nested filter when it is nor required. return true; } - if (isMappedFieldPredicate.test(field) == false) { + if (searchExecutionContext.isFieldMapped(field) == false) { // field does not exist return false; } - return nestedLookup.getNestedParent(field) != null; + return searchExecutionContext.nestedLookup().getNestedParent(field) != null; } /** Returns true if the given query might match parent documents or documents * that are nested under a different path. */ - public boolean mightMatchNonNestedDocs(Query query, String nestedPath) { + public static boolean mightMatchNonNestedDocs(Query query, String nestedPath, SearchExecutionContext searchExecutionContext) { if (query instanceof ConstantScoreQuery) { - return mightMatchNonNestedDocs(((ConstantScoreQuery) query).getQuery(), nestedPath); + return mightMatchNonNestedDocs(((ConstantScoreQuery) query).getQuery(), nestedPath, searchExecutionContext); } else if (query instanceof BoostQuery) { - return mightMatchNonNestedDocs(((BoostQuery) query).getQuery(), nestedPath); + return mightMatchNonNestedDocs(((BoostQuery) query).getQuery(), nestedPath, searchExecutionContext); } else if (query instanceof MatchAllDocsQuery) { return true; } else if (query instanceof MatchNoDocsQuery) { return false; } else if (query instanceof TermQuery) { - return mightMatchNonNestedDocs(((TermQuery) query).getTerm().field(), nestedPath); + return mightMatchNonNestedDocs(searchExecutionContext, ((TermQuery) query).getTerm().field(), nestedPath); } else if (query instanceof TermInSetQuery tis) { if (tis.getTermsCount() > 0) { - return mightMatchNonNestedDocs(tis.getField(), nestedPath); + return mightMatchNonNestedDocs(searchExecutionContext, tis.getField(), nestedPath); } else { return false; } } else if (query instanceof PointRangeQuery) { - return mightMatchNonNestedDocs(((PointRangeQuery) query).getField(), nestedPath); + return mightMatchNonNestedDocs(searchExecutionContext, ((PointRangeQuery) query).getField(), nestedPath); } else if (query instanceof IndexOrDocValuesQuery) { - return mightMatchNonNestedDocs(((IndexOrDocValuesQuery) query).getIndexQuery(), nestedPath); + return mightMatchNonNestedDocs(((IndexOrDocValuesQuery) query).getIndexQuery(), nestedPath, searchExecutionContext); } else if (query instanceof final BooleanQuery bq) { final boolean hasRequiredClauses = bq.clauses().stream().anyMatch(BooleanClause::isRequired); if (hasRequiredClauses) { @@ -131,13 +123,13 @@ public boolean mightMatchNonNestedDocs(Query query, String nestedPath) { .stream() .filter(BooleanClause::isRequired) .map(BooleanClause::query) - .allMatch(q -> mightMatchNonNestedDocs(q, nestedPath)); + .allMatch(q -> mightMatchNonNestedDocs(q, nestedPath, searchExecutionContext)); } else { return bq.clauses() .stream() .filter(c -> c.occur() == Occur.SHOULD) .map(BooleanClause::query) - .anyMatch(q -> mightMatchNonNestedDocs(q, nestedPath)); + .anyMatch(q -> mightMatchNonNestedDocs(q, nestedPath, searchExecutionContext)); } } else { return true; @@ -146,7 +138,7 @@ public boolean mightMatchNonNestedDocs(Query query, String nestedPath) { /** Returns true if a query on the given field might match parent documents * or documents that are nested under a different path. */ - boolean mightMatchNonNestedDocs(String field, String nestedPath) { + private static boolean mightMatchNonNestedDocs(SearchExecutionContext searchExecutionContext, String field, String nestedPath) { if (field.startsWith("_")) { // meta field. Every meta field behaves differently, eg. nested // documents have the same _uid as their parent, put their path in @@ -155,9 +147,10 @@ boolean mightMatchNonNestedDocs(String field, String nestedPath) { // we might add a nested filter when it is nor required. return true; } - if (isMappedFieldPredicate.test(field) == false) { + if (searchExecutionContext.isFieldMapped(field) == false) { return false; } + var nestedLookup = searchExecutionContext.nestedLookup(); String nestedParent = nestedLookup.getNestedParent(field); if (nestedParent == null || nestedParent.startsWith(nestedPath) == false) { // the field is not a sub field of the nested path diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 8ac35f7c40caa..b87d097413b67 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -444,10 +444,9 @@ public void preProcess() { public Query buildFilteredQuery(Query query) { List filters = new ArrayList<>(); NestedLookup nestedLookup = searchExecutionContext.nestedLookup(); - NestedHelper nestedHelper = new NestedHelper(nestedLookup, searchExecutionContext::isFieldMapped); if (nestedLookup != NestedLookup.EMPTY - && nestedHelper.mightMatchNestedDocs(query) - && (aliasFilter == null || nestedHelper.mightMatchNestedDocs(aliasFilter))) { + && NestedHelper.mightMatchNestedDocs(query, searchExecutionContext) + && (aliasFilter == null || NestedHelper.mightMatchNestedDocs(aliasFilter, searchExecutionContext))) { filters.add(Queries.newNonNestedFilter(searchExecutionContext.indexVersionCreated())); } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index deb7e6bd035b8..5dd2cbf32dd12 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -481,10 +481,9 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { } parentBitSet = context.bitsetFilter(parentFilter); if (filterQuery != null) { - NestedHelper nestedHelper = new NestedHelper(context.nestedLookup(), context::isFieldMapped); // We treat the provided filter as a filter over PARENT documents, so if it might match nested documents // we need to adjust it. - if (nestedHelper.mightMatchNestedDocs(filterQuery)) { + if (NestedHelper.mightMatchNestedDocs(filterQuery, context)) { // Ensure that the query only returns parent documents matching `filterQuery` filterQuery = Queries.filtered(filterQuery, parentFilter); } diff --git a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java index a7a1d33badf25..b2583eb176deb 100644 --- a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java @@ -17,6 +17,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -27,12 +28,15 @@ import java.io.IOException; import java.util.Collections; +import static java.util.Collections.emptyMap; import static org.mockito.Mockito.mock; public class NestedHelperTests extends MapperServiceTestCase { MapperService mapperService; + SearchExecutionContext searchExecutionContext; + @Override public void setUp() throws Exception { super.setUp(); @@ -68,167 +72,185 @@ public void setUp() throws Exception { } } """; mapperService = createMapperService(mapping); - } - - private static NestedHelper buildNestedHelper(MapperService mapperService) { - return new NestedHelper(mapperService.mappingLookup().nestedLookup(), field -> mapperService.fieldType(field) != null); + searchExecutionContext = new SearchExecutionContext( + 0, + 0, + mapperService.getIndexSettings(), + null, + null, + mapperService, + mapperService.mappingLookup(), + null, + null, + parserConfig(), + writableRegistry(), + null, + null, + System::currentTimeMillis, + null, + null, + () -> true, + null, + emptyMap(), + MapperMetrics.NOOP + ); } public void testMatchAll() { - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(new MatchAllDocsQuery())); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(new MatchAllDocsQuery(), "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(new MatchAllDocsQuery(), "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(new MatchAllDocsQuery(), "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(new MatchAllDocsQuery(), "nested_missing")); + assertTrue(NestedHelper.mightMatchNestedDocs(new MatchAllDocsQuery(), searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(new MatchAllDocsQuery(), "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(new MatchAllDocsQuery(), "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(new MatchAllDocsQuery(), "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(new MatchAllDocsQuery(), "nested_missing", searchExecutionContext)); } public void testMatchNo() { - assertFalse(buildNestedHelper(mapperService).mightMatchNestedDocs(new MatchNoDocsQuery())); - assertFalse(buildNestedHelper(mapperService).mightMatchNonNestedDocs(new MatchNoDocsQuery(), "nested1")); - assertFalse(buildNestedHelper(mapperService).mightMatchNonNestedDocs(new MatchNoDocsQuery(), "nested2")); - assertFalse(buildNestedHelper(mapperService).mightMatchNonNestedDocs(new MatchNoDocsQuery(), "nested3")); - assertFalse(buildNestedHelper(mapperService).mightMatchNonNestedDocs(new MatchNoDocsQuery(), "nested_missing")); + assertFalse(NestedHelper.mightMatchNestedDocs(new MatchNoDocsQuery(), searchExecutionContext)); + assertFalse(NestedHelper.mightMatchNonNestedDocs(new MatchNoDocsQuery(), "nested1", searchExecutionContext)); + assertFalse(NestedHelper.mightMatchNonNestedDocs(new MatchNoDocsQuery(), "nested2", searchExecutionContext)); + assertFalse(NestedHelper.mightMatchNonNestedDocs(new MatchNoDocsQuery(), "nested3", searchExecutionContext)); + assertFalse(NestedHelper.mightMatchNonNestedDocs(new MatchNoDocsQuery(), "nested_missing", searchExecutionContext)); } public void testTermsQuery() { Query termsQuery = mapperService.fieldType("foo").termsQuery(Collections.singletonList("bar"), null); - assertFalse(buildNestedHelper(mapperService).mightMatchNestedDocs(termsQuery)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing")); + assertFalse(NestedHelper.mightMatchNestedDocs(termsQuery, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested_missing", searchExecutionContext)); termsQuery = mapperService.fieldType("nested1.foo").termsQuery(Collections.singletonList("bar"), null); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(termsQuery)); - assertFalse(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing")); + assertTrue(NestedHelper.mightMatchNestedDocs(termsQuery, searchExecutionContext)); + assertFalse(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested_missing", searchExecutionContext)); termsQuery = mapperService.fieldType("nested2.foo").termsQuery(Collections.singletonList("bar"), null); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(termsQuery)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing")); + assertTrue(NestedHelper.mightMatchNestedDocs(termsQuery, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested_missing", searchExecutionContext)); termsQuery = mapperService.fieldType("nested3.foo").termsQuery(Collections.singletonList("bar"), null); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(termsQuery)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing")); + assertTrue(NestedHelper.mightMatchNestedDocs(termsQuery, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termsQuery, "nested_missing", searchExecutionContext)); } public void testTermQuery() { Query termQuery = mapperService.fieldType("foo").termQuery("bar", null); - assertFalse(buildNestedHelper(mapperService).mightMatchNestedDocs(termQuery)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested_missing")); + assertFalse(NestedHelper.mightMatchNestedDocs(termQuery, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested_missing", searchExecutionContext)); termQuery = mapperService.fieldType("nested1.foo").termQuery("bar", null); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(termQuery)); - assertFalse(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested_missing")); + assertTrue(NestedHelper.mightMatchNestedDocs(termQuery, searchExecutionContext)); + assertFalse(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested_missing", searchExecutionContext)); termQuery = mapperService.fieldType("nested2.foo").termQuery("bar", null); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(termQuery)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested_missing")); + assertTrue(NestedHelper.mightMatchNestedDocs(termQuery, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested_missing", searchExecutionContext)); termQuery = mapperService.fieldType("nested3.foo").termQuery("bar", null); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(termQuery)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(termQuery, "nested_missing")); + assertTrue(NestedHelper.mightMatchNestedDocs(termQuery, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(termQuery, "nested_missing", searchExecutionContext)); } public void testRangeQuery() { SearchExecutionContext context = mock(SearchExecutionContext.class); Query rangeQuery = mapperService.fieldType("foo2").rangeQuery(2, 5, true, true, null, null, null, context); - assertFalse(buildNestedHelper(mapperService).mightMatchNestedDocs(rangeQuery)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested_missing")); + assertFalse(NestedHelper.mightMatchNestedDocs(rangeQuery, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested_missing", searchExecutionContext)); rangeQuery = mapperService.fieldType("nested1.foo2").rangeQuery(2, 5, true, true, null, null, null, context); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(rangeQuery)); - assertFalse(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested_missing")); + assertTrue(NestedHelper.mightMatchNestedDocs(rangeQuery, searchExecutionContext)); + assertFalse(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested_missing", searchExecutionContext)); rangeQuery = mapperService.fieldType("nested2.foo2").rangeQuery(2, 5, true, true, null, null, null, context); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(rangeQuery)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested_missing")); + assertTrue(NestedHelper.mightMatchNestedDocs(rangeQuery, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested_missing", searchExecutionContext)); rangeQuery = mapperService.fieldType("nested3.foo2").rangeQuery(2, 5, true, true, null, null, null, context); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(rangeQuery)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(rangeQuery, "nested_missing")); + assertTrue(NestedHelper.mightMatchNestedDocs(rangeQuery, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(rangeQuery, "nested_missing", searchExecutionContext)); } public void testDisjunction() { BooleanQuery bq = new BooleanQuery.Builder().add(new TermQuery(new Term("foo", "bar")), Occur.SHOULD) .add(new TermQuery(new Term("foo", "baz")), Occur.SHOULD) .build(); - assertFalse(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1")); + assertFalse(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested1", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new TermQuery(new Term("nested1.foo", "bar")), Occur.SHOULD) .add(new TermQuery(new Term("nested1.foo", "baz")), Occur.SHOULD) .build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertFalse(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertFalse(NestedHelper.mightMatchNonNestedDocs(bq, "nested1", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new TermQuery(new Term("nested2.foo", "bar")), Occur.SHOULD) .add(new TermQuery(new Term("nested2.foo", "baz")), Occur.SHOULD) .build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested2")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested2", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new TermQuery(new Term("nested3.foo", "bar")), Occur.SHOULD) .add(new TermQuery(new Term("nested3.foo", "baz")), Occur.SHOULD) .build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested3")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested3", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new TermQuery(new Term("foo", "bar")), Occur.SHOULD) .add(new MatchAllDocsQuery(), Occur.SHOULD) .build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested1", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new TermQuery(new Term("nested1.foo", "bar")), Occur.SHOULD) .add(new MatchAllDocsQuery(), Occur.SHOULD) .build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested1", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new TermQuery(new Term("nested2.foo", "bar")), Occur.SHOULD) .add(new MatchAllDocsQuery(), Occur.SHOULD) .build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested2")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested2", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new TermQuery(new Term("nested3.foo", "bar")), Occur.SHOULD) .add(new MatchAllDocsQuery(), Occur.SHOULD) .build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested3")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested3", searchExecutionContext)); } private static Occur requiredOccur() { @@ -239,42 +261,42 @@ public void testConjunction() { BooleanQuery bq = new BooleanQuery.Builder().add(new TermQuery(new Term("foo", "bar")), requiredOccur()) .add(new MatchAllDocsQuery(), requiredOccur()) .build(); - assertFalse(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1")); + assertFalse(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested1", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new TermQuery(new Term("nested1.foo", "bar")), requiredOccur()) .add(new MatchAllDocsQuery(), requiredOccur()) .build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertFalse(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertFalse(NestedHelper.mightMatchNonNestedDocs(bq, "nested1", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new TermQuery(new Term("nested2.foo", "bar")), requiredOccur()) .add(new MatchAllDocsQuery(), requiredOccur()) .build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested2")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested2", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new TermQuery(new Term("nested3.foo", "bar")), requiredOccur()) .add(new MatchAllDocsQuery(), requiredOccur()) .build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested3")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested3", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new MatchAllDocsQuery(), requiredOccur()).add(new MatchAllDocsQuery(), requiredOccur()).build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested1", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new MatchAllDocsQuery(), requiredOccur()).add(new MatchAllDocsQuery(), requiredOccur()).build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested1")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested1", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new MatchAllDocsQuery(), requiredOccur()).add(new MatchAllDocsQuery(), requiredOccur()).build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested2")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested2", searchExecutionContext)); bq = new BooleanQuery.Builder().add(new MatchAllDocsQuery(), requiredOccur()).add(new MatchAllDocsQuery(), requiredOccur()).build(); - assertTrue(buildNestedHelper(mapperService).mightMatchNestedDocs(bq)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(bq, "nested3")); + assertTrue(NestedHelper.mightMatchNestedDocs(bq, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(bq, "nested3", searchExecutionContext)); } public void testNested() throws IOException { @@ -288,11 +310,11 @@ public void testNested() throws IOException { .build(); assertEquals(expectedChildQuery, query.getChildQuery()); - assertFalse(buildNestedHelper(mapperService).mightMatchNestedDocs(query)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested_missing")); + assertFalse(NestedHelper.mightMatchNestedDocs(query, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested_missing", searchExecutionContext)); queryBuilder = new NestedQueryBuilder("nested1", new TermQueryBuilder("nested1.foo", "bar"), ScoreMode.Avg); query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context); @@ -301,11 +323,11 @@ public void testNested() throws IOException { expectedChildQuery = new TermQuery(new Term("nested1.foo", "bar")); assertEquals(expectedChildQuery, query.getChildQuery()); - assertFalse(buildNestedHelper(mapperService).mightMatchNestedDocs(query)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested_missing")); + assertFalse(NestedHelper.mightMatchNestedDocs(query, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested_missing", searchExecutionContext)); queryBuilder = new NestedQueryBuilder("nested2", new TermQueryBuilder("nested2.foo", "bar"), ScoreMode.Avg); query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context); @@ -316,11 +338,11 @@ public void testNested() throws IOException { .build(); assertEquals(expectedChildQuery, query.getChildQuery()); - assertFalse(buildNestedHelper(mapperService).mightMatchNestedDocs(query)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested_missing")); + assertFalse(NestedHelper.mightMatchNestedDocs(query, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested_missing", searchExecutionContext)); queryBuilder = new NestedQueryBuilder("nested3", new TermQueryBuilder("nested3.foo", "bar"), ScoreMode.Avg); query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context); @@ -331,10 +353,10 @@ public void testNested() throws IOException { .build(); assertEquals(expectedChildQuery, query.getChildQuery()); - assertFalse(buildNestedHelper(mapperService).mightMatchNestedDocs(query)); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested1")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested2")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested3")); - assertTrue(buildNestedHelper(mapperService).mightMatchNonNestedDocs(query, "nested_missing")); + assertFalse(NestedHelper.mightMatchNestedDocs(query, searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested1", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested2", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested3", searchExecutionContext)); + assertTrue(NestedHelper.mightMatchNonNestedDocs(query, "nested_missing", searchExecutionContext)); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java index 14ecf4cb0d6e9..24f0a52436203 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java @@ -160,10 +160,8 @@ private static void buildRoleQuery( failIfQueryUsesClient(queryBuilder, context); Query roleQuery = context.toQuery(queryBuilder).query(); filter.add(roleQuery, SHOULD); - NestedLookup nestedLookup = context.nestedLookup(); - if (nestedLookup != NestedLookup.EMPTY) { - NestedHelper nestedHelper = new NestedHelper(nestedLookup, context::isFieldMapped); - if (nestedHelper.mightMatchNestedDocs(roleQuery)) { + if (context.nestedLookup() != NestedLookup.EMPTY) { + if (NestedHelper.mightMatchNestedDocs(roleQuery, context)) { roleQuery = new BooleanQuery.Builder().add(roleQuery, FILTER) .add(Queries.newNonNestedFilter(context.indexVersionCreated()), FILTER) .build(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 39e2a3bc1d5af..17468f7afec1b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -298,15 +298,11 @@ public SourceLoader newSourceLoader() { @Override public Query toQuery(QueryBuilder queryBuilder) { Query query = ctx.toQuery(queryBuilder).query(); - NestedLookup nestedLookup = ctx.nestedLookup(); - if (nestedLookup != NestedLookup.EMPTY) { - NestedHelper nestedHelper = new NestedHelper(nestedLookup, ctx::isFieldMapped); - if (nestedHelper.mightMatchNestedDocs(query)) { - // filter out nested documents - query = new BooleanQuery.Builder().add(query, BooleanClause.Occur.MUST) - .add(newNonNestedFilter(ctx.indexVersionCreated()), BooleanClause.Occur.FILTER) - .build(); - } + if (ctx.nestedLookup() != NestedLookup.EMPTY && NestedHelper.mightMatchNestedDocs(query, ctx)) { + // filter out nested documents + query = new BooleanQuery.Builder().add(query, BooleanClause.Occur.MUST) + .add(newNonNestedFilter(ctx.indexVersionCreated()), BooleanClause.Occur.FILTER) + .build(); } if (aliasFilter != AliasFilter.EMPTY) { Query filterQuery = ctx.toQuery(aliasFilter.getQueryBuilder()).query(); From 949816f8585982e8b38f0a3433ffb1270e56e9ff Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 5 Dec 2024 18:35:41 +0200 Subject: [PATCH 051/204] SearchServiceTests.testParseSourceValidation failure (#117963) Remove test for deprecated code removed for v_9 --- docs/changelog/117963.yaml | 5 + muted-tests.yml | 3 - .../search/SearchServiceTests.java | 115 ------------------ 3 files changed, 5 insertions(+), 118 deletions(-) create mode 100644 docs/changelog/117963.yaml diff --git a/docs/changelog/117963.yaml b/docs/changelog/117963.yaml new file mode 100644 index 0000000000000..4a50dc175786b --- /dev/null +++ b/docs/changelog/117963.yaml @@ -0,0 +1,5 @@ +pr: 117963 +summary: '`SearchServiceTests.testParseSourceValidation` failure' +area: Search +type: bug +issues: [] diff --git a/muted-tests.yml b/muted-tests.yml index b8d82f00bc43f..735a34a3b45db 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -103,9 +103,6 @@ tests: - class: org.elasticsearch.search.StressSearchServiceReaperIT method: testStressReaper issue: https://github.com/elastic/elasticsearch/issues/115816 -- class: org.elasticsearch.search.SearchServiceTests - method: testParseSourceValidation - issue: https://github.com/elastic/elasticsearch/issues/115936 - class: org.elasticsearch.xpack.application.connector.ConnectorIndexServiceTests issue: https://github.com/elastic/elasticsearch/issues/116087 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index d1ccfcbe78732..89fd25f638e1c 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -95,7 +95,6 @@ import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.dfs.AggregatedDfs; import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.fetch.ShardFetchRequest; @@ -124,7 +123,6 @@ import org.elasticsearch.search.rank.feature.RankFeatureResult; import org.elasticsearch.search.rank.feature.RankFeatureShardRequest; import org.elasticsearch.search.rank.feature.RankFeatureShardResult; -import org.elasticsearch.search.slice.SliceBuilder; import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.tasks.TaskCancelHelper; import org.elasticsearch.tasks.TaskCancelledException; @@ -2930,119 +2928,6 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { } } - /** - * This method tests validation that happens on the data nodes, which is now performed on the coordinating node. - * We still need the validation to cover for mixed cluster scenarios where the coordinating node does not perform the check yet. - */ - public void testParseSourceValidation() { - String index = randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT); - IndexService indexService = createIndex(index); - final SearchService service = getInstanceFromNode(SearchService.class); - { - // scroll and search_after - SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder()); - searchRequest.scroll(new TimeValue(1000)); - searchRequest.source().searchAfter(new String[] { "value" }); - assertCreateContextValidation(searchRequest, "`search_after` cannot be used in a scroll context.", indexService, service); - } - { - // scroll and collapse - SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder()); - searchRequest.scroll(new TimeValue(1000)); - searchRequest.source().collapse(new CollapseBuilder("field")); - assertCreateContextValidation(searchRequest, "cannot use `collapse` in a scroll context", indexService, service); - } - { - // search_after and `from` isn't valid - SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder()); - searchRequest.source().searchAfter(new String[] { "value" }); - searchRequest.source().from(10); - assertCreateContextValidation( - searchRequest, - "`from` parameter must be set to 0 when `search_after` is used", - indexService, - service - ); - } - { - // slice without scroll or pit - SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder()); - searchRequest.source().slice(new SliceBuilder(1, 10)); - assertCreateContextValidation( - searchRequest, - "[slice] can only be used with [scroll] or [point-in-time] requests", - indexService, - service - ); - } - { - // stored fields disabled with _source requested - SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder()); - searchRequest.source().storedField("_none_"); - searchRequest.source().fetchSource(true); - assertCreateContextValidation( - searchRequest, - "[stored_fields] cannot be disabled if [_source] is requested", - indexService, - service - ); - } - { - // stored fields disabled with fetch fields requested - SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder()); - searchRequest.source().storedField("_none_"); - searchRequest.source().fetchSource(false); - searchRequest.source().fetchField("field"); - assertCreateContextValidation( - searchRequest, - "[stored_fields] cannot be disabled when using the [fields] option", - indexService, - service - ); - } - } - - private static void assertCreateContextValidation( - SearchRequest searchRequest, - String errorMessage, - IndexService indexService, - SearchService searchService - ) { - ShardId shardId = new ShardId(indexService.index(), 0); - long nowInMillis = System.currentTimeMillis(); - String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(3, 10); - searchRequest.allowPartialSearchResults(randomBoolean()); - ShardSearchRequest request = new ShardSearchRequest( - OriginalIndices.NONE, - searchRequest, - shardId, - 0, - indexService.numberOfShards(), - AliasFilter.EMPTY, - 1f, - nowInMillis, - clusterAlias - ); - - SearchShardTask task = new SearchShardTask(1, "type", "action", "description", null, emptyMap()); - - ReaderContext readerContext = null; - try { - ReaderContext createOrGetReaderContext = searchService.createOrGetReaderContext(request); - readerContext = createOrGetReaderContext; - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> searchService.createContext(createOrGetReaderContext, request, task, ResultsType.QUERY, randomBoolean()) - ); - assertThat(exception.getMessage(), containsString(errorMessage)); - } finally { - if (readerContext != null) { - readerContext.close(); - searchService.freeReaderContext(readerContext.id()); - } - } - } - private static ReaderContext createReaderContext(IndexService indexService, IndexShard indexShard) { return new ReaderContext( new ShardSearchContextId(UUIDs.randomBase64UUID(), randomNonNegativeLong()), From 176bf7a85abb2e2af06f60901b448bef945d528b Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 5 Dec 2024 11:53:15 -0500 Subject: [PATCH 052/204] ESQL Javadoc for creating new data types (#117520) This adds some java doc to the DataType enum, listing out the steps I followed for adding DateNanos. Hopefully it's helpful to future folks adding data types. --------- Co-authored-by: Bogdan Pintea --- .../xpack/esql/core/type/DataType.java | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index a63571093ba58..d86cdb0de038c 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -32,6 +32,113 @@ import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck; import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck; +/** + * This enum represents data types the ES|QL query processing layer is able to + * interact with in some way. This includes fully representable types (e.g. + * {@link DataType#LONG}, numeric types which we promote (e.g. {@link DataType#SHORT}) + * or fold into other types (e.g. {@link DataType#DATE_PERIOD}) early in the + * processing pipeline, types for internal use + * cases (e.g. {@link DataType#PARTIAL_AGG}), and types which the language + * doesn't support, but require special handling anyway (e.g. + * {@link DataType#OBJECT}) + * + *

Process for adding a new data type

+ * Note: it is not expected that all the following steps be done in a single PR. + * Use capabilities to gate tests as you go, and use as many PRs as you think + * appropriate. New data types are complex, and smaller PRs will make reviews + * easier. + *
    + *
  • + * Create a new feature flag for the type in {@link EsqlCorePlugin}. We + * recommend developing the data type over a series of smaller PRs behind + * a feature flag; even for relatively simple data types.
    • + *
  • + * Add a capability to EsqlCapabilities related to the new type, and + * gated by the feature flag you just created. Again, using the feature + * flag is preferred over snapshot-only. As development progresses, you may + * need to add more capabilities related to the new type, e.g. for + * supporting specific functions. This is fine, and expected.
    • + *
  • + * Create a new CSV test file for the new type. You'll either need to + * create a new data file as well, or add values of the new type to + * and existing data file. See CsvTestDataLoader for creating a new data + * set.
    • + *
  • + * In the new CSV test file, start adding basic functionality tests. + * These should include reading and returning values, both from indexed data + * and from the ROW command. It should also include functions that support + * "every" type, such as Case or MvFirst.
    • + *
  • + * Add the new type to the CsvTestUtils#Type enum, if it isn't already + * there. You also need to modify CsvAssert to support reading values + * of the new type.
    • + *
  • + * At this point, the CSV tests should fail with a sensible ES|QL error + * message. Make sure they're failing in ES|QL, not in the test + * framework.
    • + *
  • + * Add the new data type to this enum. This will cause a bunch of + * compile errors for switch statements throughout the code. Resolve those + * as appropriate. That is the main way in which the new type will be tied + * into the framework.
    • + *
  • + * Add the new type to the {@link DataType#UNDER_CONSTRUCTION} + * collection. This is used by the test framework to disable some checks + * around how functions report their supported types, which would otherwise + * generate a lot of noise while the type is still in development.
    • + *
  • + * Add typed data generators to TestCaseSupplier, and make sure all + * functions that support the new type have tests for it.
    • + *
  • + * Work to support things all types should do. Equality and the + * "typeless" MV functions (MvFirst, MvLast, and MvCount) should work for + * most types. Case and Coalesce should also support all types. + * If the type has a natural ordering, make sure to test + * sorting and the other binary comparisons. Make sure these functions all + * have CSV tests that run against indexed data.
    • + *
  • + * Add conversion functions as appropriate. Almost all types should + * support ToString, and should have a "ToType" function that accepts a + * string. There may be other logical conversions depending on the nature + * of the type. Make sure to add the conversion function to the + * TYPE_TO_CONVERSION_FUNCTION map in EsqlDataTypeConverter. Make sure the + * conversion functions have CSV tests that run against indexed data.
    • + *
  • + * Support the new type in aggregations that are type independent. + * This includes Values, Count, and Count Distinct. Make sure there are + * CSV tests against indexed data for these.
    • + *
  • + * Support other functions and aggregations as appropriate, making sure + * to included CSV tests.
    • + *
  • + * Consider how the type will interact with other types. For example, + * if the new type is numeric, it may be good for it to be comparable with + * other numbers. Supporting this may require new logic in + * EsqlDataTypeConverter#commonType, individual function type checking, the + * verifier rules, or other places. We suggest starting with CSV tests and + * seeing where they fail.
    • + *
+ * There are some additional steps that should be taken when removing the + * feature flag and getting ready for a release: + *
    + *
  • + * Ensure the capabilities for this type are always enabled + *
    • + *
  • + * Remove the type from the {@link DataType#UNDER_CONSTRUCTION} + * collection
    • + *
  • + * Fix new test failures related to declared function types + *
    • + *
  • + * Make sure to run the full test suite locally via gradle to generate + * the function type tables and helper files with the new type. Ensure all + * the functions that support the type have appropriate docs for it.
    • + *
  • + * If appropriate, remove the type from the ESQL limitations list of + * unsupported types.
    • + *
+ */ public enum DataType { /** * Fields of this type are unsupported by any functions and are always From 162140e1d2e1c82faa5eada4d97a1143ba89afde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Thu, 5 Dec 2024 17:58:56 +0100 Subject: [PATCH 053/204] Close URLClassLoaders to make Windows happy deleting the temp test jar files (#118083) --- .../bootstrap/PluginsResolverTests.java | 67 ++++++++++--------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/bootstrap/PluginsResolverTests.java b/server/src/test/java/org/elasticsearch/bootstrap/PluginsResolverTests.java index 331f0f7ad13e9..798b576500d72 100644 --- a/server/src/test/java/org/elasticsearch/bootstrap/PluginsResolverTests.java +++ b/server/src/test/java/org/elasticsearch/bootstrap/PluginsResolverTests.java @@ -136,25 +136,28 @@ public void testResolveMultipleNonModularPlugins() throws IOException, ClassNotF Path jar1 = createNonModularPluginJar(home, "plugin1", "p", "A"); Path jar2 = createNonModularPluginJar(home, "plugin2", "q", "B"); - var loader1 = createClassLoader(jar1); - var loader2 = createClassLoader(jar2); - - PluginBundle bundle1 = createMockBundle("plugin1", null, "p.A"); - PluginBundle bundle2 = createMockBundle("plugin2", null, "q.B"); - PluginsLoader mockPluginsLoader = mock(PluginsLoader.class); - - when(mockPluginsLoader.pluginLayers()).thenReturn( - Stream.of(new TestPluginLayer(bundle1, loader1, ModuleLayer.boot()), new TestPluginLayer(bundle2, loader2, ModuleLayer.boot())) - ); - PluginsResolver pluginsResolver = PluginsResolver.create(mockPluginsLoader); - - var testClass1 = loader1.loadClass("p.A"); - var testClass2 = loader2.loadClass("q.B"); - var resolvedPluginName1 = pluginsResolver.resolveClassToPluginName(testClass1); - var resolvedPluginName2 = pluginsResolver.resolveClassToPluginName(testClass2); - - assertEquals("plugin1", resolvedPluginName1); - assertEquals("plugin2", resolvedPluginName2); + try (var loader1 = createClassLoader(jar1); var loader2 = createClassLoader(jar2)) { + + PluginBundle bundle1 = createMockBundle("plugin1", null, "p.A"); + PluginBundle bundle2 = createMockBundle("plugin2", null, "q.B"); + PluginsLoader mockPluginsLoader = mock(PluginsLoader.class); + + when(mockPluginsLoader.pluginLayers()).thenReturn( + Stream.of( + new TestPluginLayer(bundle1, loader1, ModuleLayer.boot()), + new TestPluginLayer(bundle2, loader2, ModuleLayer.boot()) + ) + ); + PluginsResolver pluginsResolver = PluginsResolver.create(mockPluginsLoader); + + var testClass1 = loader1.loadClass("p.A"); + var testClass2 = loader2.loadClass("q.B"); + var resolvedPluginName1 = pluginsResolver.resolveClassToPluginName(testClass1); + var resolvedPluginName2 = pluginsResolver.resolveClassToPluginName(testClass2); + + assertEquals("plugin1", resolvedPluginName1); + assertEquals("plugin2", resolvedPluginName2); + } } public void testResolveNonModularPlugin() throws IOException, ClassNotFoundException { @@ -164,22 +167,22 @@ public void testResolveNonModularPlugin() throws IOException, ClassNotFoundExcep Path jar = createNonModularPluginJar(home, pluginName, "p", "A"); - var loader = createClassLoader(jar); - - PluginBundle bundle = createMockBundle(pluginName, null, "p.A"); - PluginsLoader mockPluginsLoader = mock(PluginsLoader.class); + try (var loader = createClassLoader(jar)) { + PluginBundle bundle = createMockBundle(pluginName, null, "p.A"); + PluginsLoader mockPluginsLoader = mock(PluginsLoader.class); - when(mockPluginsLoader.pluginLayers()).thenReturn(Stream.of(new TestPluginLayer(bundle, loader, ModuleLayer.boot()))); - PluginsResolver pluginsResolver = PluginsResolver.create(mockPluginsLoader); + when(mockPluginsLoader.pluginLayers()).thenReturn(Stream.of(new TestPluginLayer(bundle, loader, ModuleLayer.boot()))); + PluginsResolver pluginsResolver = PluginsResolver.create(mockPluginsLoader); - var testClass = loader.loadClass("p.A"); - var resolvedPluginName = pluginsResolver.resolveClassToPluginName(testClass); - var unresolvedPluginName1 = pluginsResolver.resolveClassToPluginName(PluginsResolver.class); - var unresolvedPluginName2 = pluginsResolver.resolveClassToPluginName(String.class); + var testClass = loader.loadClass("p.A"); + var resolvedPluginName = pluginsResolver.resolveClassToPluginName(testClass); + var unresolvedPluginName1 = pluginsResolver.resolveClassToPluginName(PluginsResolver.class); + var unresolvedPluginName2 = pluginsResolver.resolveClassToPluginName(String.class); - assertEquals(pluginName, resolvedPluginName); - assertNull(unresolvedPluginName1); - assertNull(unresolvedPluginName2); + assertEquals(pluginName, resolvedPluginName); + assertNull(unresolvedPluginName1); + assertNull(unresolvedPluginName2); + } } private static URLClassLoader createClassLoader(Path jar) throws MalformedURLException { From eb4f33ae7be785404a97f6ca9c42f94749de3599 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 6 Dec 2024 04:09:50 +1100 Subject: [PATCH 054/204] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT #117981 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 735a34a3b45db..a09e46415fdc1 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -251,6 +251,8 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveTests method: test40AutoconfigurationNotTriggeredWhenNodeIsMeantToJoinExistingCluster issue: https://github.com/elastic/elasticsearch/issues/118029 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + issue: https://github.com/elastic/elasticsearch/issues/117981 # Examples: # From 21f72f8f6a9e2518a21631e61713880aa027c2c4 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Thu, 5 Dec 2024 12:15:48 -0500 Subject: [PATCH 055/204] Removing dead/unused deprecation logger code (#118082) --- .../rest/action/document/RestGetSourceAction.java | 4 ---- .../rest/action/document/RestMultiTermVectorsAction.java | 3 --- .../org/elasticsearch/rest/action/search/RestCountAction.java | 3 --- .../elasticsearch/rest/action/search/RestSearchAction.java | 3 --- .../org/elasticsearch/search/builder/SearchSourceBuilder.java | 2 -- .../org/elasticsearch/search/sort/GeoDistanceSortBuilder.java | 2 -- 6 files changed, 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java index a09fcbd0c5273..7e4d23db70288 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java @@ -15,7 +15,6 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; @@ -40,9 +39,6 @@ */ @ServerlessScope(Scope.PUBLIC) public class RestGetSourceAction extends BaseRestHandler { - private final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetSourceAction.class); - static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in get_source and exist_source " - + "requests is deprecated."; @Override public List routes() { diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java index 65aa1869a41e4..9d39bf7f343c6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -28,8 +27,6 @@ @ServerlessScope(Scope.PUBLIC) public class RestMultiTermVectorsAction extends BaseRestHandler { - private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestMultiTermVectorsAction.class); - static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in multi term vector requests is deprecated."; @Override public List routes() { diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java index c1a55874bfc58..b0e08b376f9d0 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java @@ -14,7 +14,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -36,8 +35,6 @@ @ServerlessScope(Scope.PUBLIC) public class RestCountAction extends BaseRestHandler { - private final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestCountAction.class); - static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in count requests is deprecated."; @Override public List routes() { diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index ff062084a3cbb..a9c2ff7576b05 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; @@ -56,8 +55,6 @@ @ServerlessScope(Scope.PUBLIC) public class RestSearchAction extends BaseRestHandler { - private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestSearchAction.class); - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in search requests is deprecated."; /** * Indicates whether hits.total should be rendered as an integer or an object diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 098a2b2f45d2f..3554a6dc08b90 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Booleans; @@ -92,7 +91,6 @@ * @see SearchRequest#source(SearchSourceBuilder) */ public final class SearchSourceBuilder implements Writeable, ToXContentObject, Rewriteable { - private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(SearchSourceBuilder.class); public static final ParseField FROM_FIELD = new ParseField("from"); public static final ParseField SIZE_FIELD = new ParseField("size"); diff --git a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index 6640f0f858404..2aaade35fb8f3 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.fielddata.FieldData; @@ -67,7 +66,6 @@ * A geo distance based sorting on a geo point like field. */ public class GeoDistanceSortBuilder extends SortBuilder { - private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(GeoDistanceSortBuilder.class); public static final String NAME = "_geo_distance"; public static final String ALTERNATIVE_NAME = "_geoDistance"; From 4740b02a9b1c7c7ae0c6383c5985720bbdfa353c Mon Sep 17 00:00:00 2001 From: Henrique Paes Date: Thu, 5 Dec 2024 12:22:48 -0500 Subject: [PATCH 056/204] Wrap jackson exception on malformed json string (#114445) This commit hides the underlying Jackson parse exception when encountered while parsing string tokens. --- docs/changelog/114445.yaml | 6 ++++++ .../xcontent/provider/json/JsonXContentParser.java | 6 +++++- .../java/org/elasticsearch/http/BulkRestIT.java | 3 +-- .../common/xcontent/json/JsonXContentTests.java | 13 +++++++++++++ 4 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/114445.yaml diff --git a/docs/changelog/114445.yaml b/docs/changelog/114445.yaml new file mode 100644 index 0000000000000..afbc080d1e0b9 --- /dev/null +++ b/docs/changelog/114445.yaml @@ -0,0 +1,6 @@ +pr: 114445 +summary: Wrap jackson exception on malformed json string +area: Infra/Core +type: bug +issues: + - 114142 diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java index d42c56845d03f..38ef8bc2e4ef0 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java @@ -108,7 +108,11 @@ public String text() throws IOException { if (currentToken().isValue() == false) { throwOnNoText(); } - return parser.getText(); + try { + return parser.getText(); + } catch (JsonParseException e) { + throw newXContentParseException(e); + } } private void throwOnNoText() { diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java index 369d0824bdb28..3faa88339f0a3 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java @@ -74,8 +74,7 @@ public void testBulkInvalidIndexNameString() throws IOException { ResponseException responseException = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request)); assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(BAD_REQUEST.getStatus())); - assertThat(responseException.getMessage(), containsString("could not parse bulk request body")); - assertThat(responseException.getMessage(), containsString("json_parse_exception")); + assertThat(responseException.getMessage(), containsString("x_content_parse_exception")); assertThat(responseException.getMessage(), containsString("Invalid UTF-8")); } diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java index 55f6cc5498d80..4135ead545e07 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java @@ -11,6 +11,9 @@ import org.elasticsearch.common.xcontent.BaseXContentTestCase; import org.elasticsearch.xcontent.XContentGenerator; +import org.elasticsearch.xcontent.XContentParseException; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; @@ -28,4 +31,14 @@ public void testBigInteger() throws Exception { XContentGenerator generator = JsonXContent.jsonXContent.createGenerator(os); doTestBigInteger(generator, os); } + + public void testMalformedJsonFieldThrowsXContentException() throws Exception { + String json = "{\"test\":\"/*/}"; + try (XContentParser parser = JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, json)) { + parser.nextToken(); + parser.nextToken(); + parser.nextToken(); + assertThrows(XContentParseException.class, () -> parser.text()); + } + } } From 7070e95fa78ef29df363d1d49cd05f0b79a835bf Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Thu, 5 Dec 2024 09:43:18 -0800 Subject: [PATCH 057/204] Update BWC version logic to support multiple bugfix versions (#117943) --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic.yml | 4 +- .ci/snapshotBwcVersions | 1 + ...lDistributionBwcSetupPluginFuncTest.groovy | 24 ++- ...lDistributionDownloadPluginFuncTest.groovy | 4 +- ...acyYamlRestCompatTestPluginFuncTest.groovy | 16 +- .../distribution/bwc/bugfix2/build.gradle | 0 .../distribution/bwc/maintenance/build.gradle | 0 .../internal/fake_git/remote/settings.gradle | 2 + .../gradle/internal/BwcVersions.java | 126 ++++++----- .../internal/info/GlobalBuildInfoPlugin.java | 25 ++- .../gradle/internal/BwcVersionsSpec.groovy | 196 +++++++++++------- ...stractDistributionDownloadPluginTests.java | 14 +- .../fixtures/AbstractGradleFuncTest.groovy | 18 +- distribution/bwc/bugfix2/build.gradle | 0 settings.gradle | 1 + 16 files changed, 273 insertions(+), 160 deletions(-) create mode 100644 build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix2/build.gradle create mode 100644 build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/maintenance/build.gradle create mode 100644 distribution/bwc/bugfix2/build.gradle diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 6c8b8edfcbac1..4bc72aec20972 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -56,7 +56,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"] + BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 69d11ef1dabb6..3d6095d0b9e63 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -448,7 +448,7 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk21 - BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"] + BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -490,7 +490,7 @@ steps: ES_RUNTIME_JAVA: - openjdk21 - openjdk23 - BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"] + BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 5514fc376a285..f92881da7fea4 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,4 +1,5 @@ BWC_VERSION: + - "8.15.6" - "8.16.2" - "8.17.0" - "8.18.0" diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy index 6d080e1c80763..bb100b6b23882 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy @@ -9,9 +9,10 @@ package org.elasticsearch.gradle.internal +import spock.lang.Unroll + import org.elasticsearch.gradle.fixtures.AbstractGitAwareGradleFuncTest import org.gradle.testkit.runner.TaskOutcome -import spock.lang.Unroll class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest { @@ -23,8 +24,10 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF apply plugin: 'elasticsearch.internal-distribution-bwc-setup' """ execute("git branch origin/8.x", file("cloned")) + execute("git branch origin/8.3", file("cloned")) + execute("git branch origin/8.2", file("cloned")) + execute("git branch origin/8.1", file("cloned")) execute("git branch origin/7.16", file("cloned")) - execute("git branch origin/7.15", file("cloned")) } def "builds distribution from branches via archives extractedAssemble"() { @@ -48,10 +51,11 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF assertOutputContains(result.output, "[$bwcDistVersion] > Task :distribution:archives:darwin-tar:${expectedAssembleTaskName}") where: - bwcDistVersion | bwcProject | expectedAssembleTaskName - "8.0.0" | "minor" | "extractedAssemble" - "7.16.0" | "staged" | "extractedAssemble" - "7.15.2" | "bugfix" | "extractedAssemble" + bwcDistVersion | bwcProject | expectedAssembleTaskName + "8.4.0" | "minor" | "extractedAssemble" + "8.3.0" | "staged" | "extractedAssemble" + "8.2.1" | "bugfix" | "extractedAssemble" + "8.1.3" | "bugfix2" | "extractedAssemble" } @Unroll @@ -70,8 +74,8 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF where: bwcDistVersion | platform - "8.0.0" | "darwin" - "8.0.0" | "linux" + "8.4.0" | "darwin" + "8.4.0" | "linux" } def "bwc expanded distribution folder can be resolved as bwc project artifact"() { @@ -107,11 +111,11 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF result.task(":resolveExpandedDistribution").outcome == TaskOutcome.SUCCESS result.task(":distribution:bwc:minor:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS and: "assemble task triggered" - result.output.contains("[8.0.0] > Task :distribution:archives:darwin-tar:extractedAssemble") + result.output.contains("[8.4.0] > Task :distribution:archives:darwin-tar:extractedAssemble") result.output.contains("expandedRootPath /distribution/bwc/minor/build/bwc/checkout-8.x/" + "distribution/archives/darwin-tar/build/install") result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.x/" + - "distribution/archives/darwin-tar/build/install/elasticsearch-8.0.0-SNAPSHOT") + "distribution/archives/darwin-tar/build/install/elasticsearch-8.4.0-SNAPSHOT") } } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy index eb6185e5aed57..fc5d432a9ef9a 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy @@ -57,7 +57,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest elasticsearch_distributions { test_distro { - version = "8.0.0" + version = "8.4.0" type = "archive" platform = "linux" architecture = Architecture.current(); @@ -87,7 +87,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest elasticsearch_distributions { test_distro { - version = "8.0.0" + version = "8.4.0" type = "archive" platform = "linux" architecture = Architecture.current(); diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy index e3efe3d7ffbf7..15b057a05e039 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy @@ -40,7 +40,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe given: internalBuild() - subProject(":distribution:bwc:staged") << """ + subProject(":distribution:bwc:minor") << """ configurations { checkout } artifacts { checkout(new File(projectDir, "checkoutDir")) @@ -61,11 +61,11 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe result.task(transformTask).outcome == TaskOutcome.NO_SOURCE } - def "yamlRestCompatTest executes and copies api and transforms tests from :bwc:staged"() { + def "yamlRestCompatTest executes and copies api and transforms tests from :bwc:minor"() { given: internalBuild() - subProject(":distribution:bwc:staged") << """ + subProject(":distribution:bwc:minor") << """ configurations { checkout } artifacts { checkout(new File(projectDir, "checkoutDir")) @@ -98,8 +98,8 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe String api = "foo.json" String test = "10_basic.yml" //add the compatible test and api files, these are the prior version's normal yaml rest tests - file("distribution/bwc/staged/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << "" - file("distribution/bwc/staged/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << "" + file("distribution/bwc/minor/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << "" + file("distribution/bwc/minor/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << "" when: def result = gradleRunner("yamlRestCompatTest").build() @@ -145,7 +145,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe given: internalBuild() withVersionCatalogue() - subProject(":distribution:bwc:staged") << """ + subProject(":distribution:bwc:minor") << """ configurations { checkout } artifacts { checkout(new File(projectDir, "checkoutDir")) @@ -186,7 +186,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe given: internalBuild() - subProject(":distribution:bwc:staged") << """ + subProject(":distribution:bwc:minor") << """ configurations { checkout } artifacts { checkout(new File(projectDir, "checkoutDir")) @@ -230,7 +230,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe setupRestResources([], []) - file("distribution/bwc/staged/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """ + file("distribution/bwc/minor/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """ "one": - do: do_.some.key_to_replace: diff --git a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix2/build.gradle b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix2/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/maintenance/build.gradle b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/maintenance/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle index 8c321294b585f..e931537fcd6e9 100644 --- a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle +++ b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle @@ -10,9 +10,11 @@ rootProject.name = "root" include ":distribution:bwc:bugfix" +include ":distribution:bwc:bugfix2" include ":distribution:bwc:minor" include ":distribution:bwc:major" include ":distribution:bwc:staged" +include ":distribution:bwc:maintenance" include ":distribution:archives:darwin-tar" include ":distribution:archives:oss-darwin-tar" include ":distribution:archives:darwin-aarch64-tar" diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java index 93c2623a23d31..37b28389ad97b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java @@ -21,14 +21,15 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; -import java.util.TreeSet; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static java.util.Collections.reverseOrder; import static java.util.Collections.unmodifiableList; +import static java.util.Comparator.comparing; /** * A container for elasticsearch supported version information used in BWC testing. @@ -73,11 +74,11 @@ public class BwcVersions implements Serializable { private final transient List versions; private final Map unreleased; - public BwcVersions(List versionLines) { - this(versionLines, Version.fromString(VersionProperties.getElasticsearch())); + public BwcVersions(List versionLines, List developmentBranches) { + this(versionLines, Version.fromString(VersionProperties.getElasticsearch()), developmentBranches); } - public BwcVersions(Version currentVersionProperty, List allVersions) { + public BwcVersions(Version currentVersionProperty, List allVersions, List developmentBranches) { if (allVersions.isEmpty()) { throw new IllegalArgumentException("Could not parse any versions"); } @@ -86,12 +87,12 @@ public BwcVersions(Version currentVersionProperty, List allVersions) { this.currentVersion = allVersions.get(allVersions.size() - 1); assertCurrentVersionMatchesParsed(currentVersionProperty); - this.unreleased = computeUnreleased(); + this.unreleased = computeUnreleased(developmentBranches); } // Visible for testing - BwcVersions(List versionLines, Version currentVersionProperty) { - this(currentVersionProperty, parseVersionLines(versionLines)); + BwcVersions(List versionLines, Version currentVersionProperty, List developmentBranches) { + this(currentVersionProperty, parseVersionLines(versionLines), developmentBranches); } private static List parseVersionLines(List versionLines) { @@ -126,58 +127,77 @@ public void forPreviousUnreleased(Consumer consumer) { getUnreleased().stream().filter(version -> version.equals(currentVersion) == false).map(unreleased::get).forEach(consumer); } - private String getBranchFor(Version version) { - if (version.equals(currentVersion)) { - // Just assume the current branch is 'main'. It's actually not important, we never check out the current branch. - return "main"; - } else { + private String getBranchFor(Version version, List developmentBranches) { + // If the current version matches a specific feature freeze branch, use that + if (developmentBranches.contains(version.getMajor() + "." + version.getMinor())) { return version.getMajor() + "." + version.getMinor(); + } else if (developmentBranches.contains(version.getMajor() + ".x")) { // Otherwise if an n.x branch exists and we are that major + return version.getMajor() + ".x"; + } else { // otherwise we're the main branch + return "main"; } } - private Map computeUnreleased() { - Set unreleased = new TreeSet<>(); - // The current version is being worked, is always unreleased - unreleased.add(currentVersion); - // Recurse for all unreleased versions starting from the current version - addUnreleased(unreleased, currentVersion, 0); + private Map computeUnreleased(List developmentBranches) { + Map result = new TreeMap<>(); - // Grab the latest version from the previous major if necessary as well, this is going to be a maintenance release - Version maintenance = versions.stream() - .filter(v -> v.getMajor() == currentVersion.getMajor() - 1) - .max(Comparator.naturalOrder()) - .orElseThrow(); - // This is considered the maintenance release only if we haven't yet encountered it - boolean hasMaintenanceRelease = unreleased.add(maintenance); + // The current version is always in development + String currentBranch = getBranchFor(currentVersion, developmentBranches); + result.put(currentVersion, new UnreleasedVersionInfo(currentVersion, currentBranch, ":distribution")); + + // Check for an n.x branch as well + if (currentBranch.equals("main") && developmentBranches.stream().anyMatch(s -> s.endsWith(".x"))) { + // This should correspond to the latest new minor + Version version = versions.stream() + .sorted(Comparator.reverseOrder()) + .filter(v -> v.getMajor() == (currentVersion.getMajor() - 1) && v.getRevision() == 0) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Unable to determine development version for branch")); + String branch = getBranchFor(version, developmentBranches); + assert branch.equals(currentVersion.getMajor() - 1 + ".x") : "Expected branch does not match development branch"; + + result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:minor")); + } - List unreleasedList = unreleased.stream().sorted(Comparator.reverseOrder()).toList(); - Map result = new TreeMap<>(); - boolean newMinor = false; - for (int i = 0; i < unreleasedList.size(); i++) { - Version esVersion = unreleasedList.get(i); - // This is either a new minor or staged release - if (currentVersion.equals(esVersion)) { - result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution")); - } else if (esVersion.getRevision() == 0) { - // If there are two upcoming unreleased minors then this one is the new minor - if (newMinor == false && unreleasedList.get(i + 1).getRevision() == 0) { - result.put(esVersion, new UnreleasedVersionInfo(esVersion, esVersion.getMajor() + ".x", ":distribution:bwc:minor")); - newMinor = true; - } else if (newMinor == false - && unreleasedList.stream().filter(v -> v.getMajor() == esVersion.getMajor() && v.getRevision() == 0).count() == 1) { - // This is the only unreleased new minor which means we've not yet staged it for release - result.put(esVersion, new UnreleasedVersionInfo(esVersion, esVersion.getMajor() + ".x", ":distribution:bwc:minor")); - newMinor = true; - } else { - result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:staged")); - } - } else { - // If this is the oldest unreleased version and we have a maintenance release - if (i == unreleasedList.size() - 1 && hasMaintenanceRelease) { - result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:maintenance")); - } else { - result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:bugfix")); - } + // Now handle all the feature freeze branches + List featureFreezeBranches = developmentBranches.stream() + .filter(b -> Pattern.matches("[0-9]+\\.[0-9]+", b)) + .sorted(reverseOrder(comparing(s -> Version.fromString(s, Version.Mode.RELAXED)))) + .toList(); + + boolean existingBugfix = false; + for (int i = 0; i < featureFreezeBranches.size(); i++) { + String branch = featureFreezeBranches.get(i); + Version version = versions.stream() + .sorted(Comparator.reverseOrder()) + .filter(v -> v.toString().startsWith(branch)) + .findFirst() + .orElse(null); + + // If we don't know about this version we can ignore it + if (version == null) { + continue; + } + + // If this is the current version we can ignore as we've already handled it + if (version.equals(currentVersion)) { + continue; + } + + // We only maintain compatibility back one major so ignore anything older + if (currentVersion.getMajor() - version.getMajor() > 1) { + continue; + } + + // This is the maintenance version + if (i == featureFreezeBranches.size() - 1) { + result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:maintenance")); + } else if (version.getRevision() == 0) { // This is the next staged minor + result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:staged")); + } else { // This is a bugfix + String project = existingBugfix ? "bugfix2" : "bugfix"; + result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:" + project)); + existingBugfix = true; } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java index 0535026b2594e..27d2a66feb206 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java @@ -8,6 +8,9 @@ */ package org.elasticsearch.gradle.internal.info; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + import org.apache.commons.io.IOUtils; import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.internal.BwcVersions; @@ -44,11 +47,13 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.UncheckedIOException; import java.nio.file.Files; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Random; @@ -68,6 +73,7 @@ public class GlobalBuildInfoPlugin implements Plugin { private final JavaInstallationRegistry javaInstallationRegistry; private final JvmMetadataDetector metadataDetector; private final ProviderFactory providers; + private final ObjectMapper objectMapper; private JavaToolchainService toolChainService; private Project project; @@ -82,7 +88,7 @@ public GlobalBuildInfoPlugin( this.javaInstallationRegistry = javaInstallationRegistry; this.metadataDetector = new ErrorTraceMetadataDetector(metadataDetector); this.providers = providers; - + this.objectMapper = new ObjectMapper(); } @Override @@ -190,12 +196,27 @@ private BwcVersions resolveBwcVersions() { ); try (var is = new FileInputStream(versionsFilePath)) { List versionLines = IOUtils.readLines(is, "UTF-8"); - return new BwcVersions(versionLines); + return new BwcVersions(versionLines, getDevelopmentBranches()); } catch (IOException e) { throw new IllegalStateException("Unable to resolve to resolve bwc versions from versionsFile.", e); } } + private List getDevelopmentBranches() { + List branches = new ArrayList<>(); + File branchesFile = new File(Util.locateElasticsearchWorkspace(project.getGradle()), "branches.json"); + try (InputStream is = new FileInputStream(branchesFile)) { + JsonNode json = objectMapper.readTree(is); + for (JsonNode node : json.get("branches")) { + branches.add(node.get("branch").asText()); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + return branches; + } + private void logGlobalBuildInfo(BuildParameterExtension buildParams) { final String osName = System.getProperty("os.name"); final String osVersion = System.getProperty("os.version"); diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/BwcVersionsSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/BwcVersionsSpec.groovy index 9c7d20d84a670..4d033564a42b4 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/BwcVersionsSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/BwcVersionsSpec.groovy @@ -17,8 +17,9 @@ import org.elasticsearch.gradle.internal.BwcVersions.UnreleasedVersionInfo class BwcVersionsSpec extends Specification { List versionLines = [] - def "current version is next minor with next major and last minor both staged"() { + def "current version is next major"() { given: + addVersion('7.17.10', '8.9.0') addVersion('8.14.0', '9.9.0') addVersion('8.14.1', '9.9.0') addVersion('8.14.2', '9.9.0') @@ -29,25 +30,25 @@ class BwcVersionsSpec extends Specification { addVersion('8.16.1', '9.10.0') addVersion('8.17.0', '9.10.0') addVersion('9.0.0', '10.0.0') - addVersion('9.1.0', '10.1.0') when: - def bwc = new BwcVersions(versionLines, v('9.1.0')) + def bwc = new BwcVersions(versionLines, v('9.0.0'), ['main', '8.x', '8.16', '8.15', '7.17']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ + (v('8.15.2')): new UnreleasedVersionInfo(v('8.15.2'), '8.15', ':distribution:bwc:bugfix2'), (v('8.16.1')): new UnreleasedVersionInfo(v('8.16.1'), '8.16', ':distribution:bwc:bugfix'), - (v('8.17.0')): new UnreleasedVersionInfo(v('8.17.0'), '8.17', ':distribution:bwc:staged'), - (v('9.0.0')): new UnreleasedVersionInfo(v('9.0.0'), '9.x', ':distribution:bwc:minor'), - (v('9.1.0')): new UnreleasedVersionInfo(v('9.1.0'), 'main', ':distribution') + (v('8.17.0')): new UnreleasedVersionInfo(v('8.17.0'), '8.x', ':distribution:bwc:minor'), + (v('9.0.0')): new UnreleasedVersionInfo(v('9.0.0'), 'main', ':distribution'), ] - bwc.wireCompatible == [v('8.17.0'), v('9.0.0'), v('9.1.0')] - bwc.indexCompatible == [v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('9.0.0'), v('9.1.0')] + bwc.wireCompatible == [v('8.17.0'), v('9.0.0')] + bwc.indexCompatible == [v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('9.0.0')] } - def "current is next minor with upcoming minor staged"() { + def "current version is next major with staged minor"() { given: + addVersion('7.17.10', '8.9.0') addVersion('8.14.0', '9.9.0') addVersion('8.14.1', '9.9.0') addVersion('8.14.2', '9.9.0') @@ -57,53 +58,106 @@ class BwcVersionsSpec extends Specification { addVersion('8.16.0', '9.10.0') addVersion('8.16.1', '9.10.0') addVersion('8.17.0', '9.10.0') - addVersion('8.17.1', '9.10.0') + addVersion('8.18.0', '9.10.0') addVersion('9.0.0', '10.0.0') - addVersion('9.1.0', '10.1.0') when: - def bwc = new BwcVersions(versionLines, v('9.1.0')) + def bwc = new BwcVersions(versionLines, v('9.0.0'), ['main', '8.x', '8.17', '8.16', '8.15', '7.17']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ - (v('8.17.1')): new UnreleasedVersionInfo(v('8.17.1'), '8.17', ':distribution:bwc:bugfix'), + (v('8.15.2')): new UnreleasedVersionInfo(v('8.15.2'), '8.15', ':distribution:bwc:bugfix2'), + (v('8.16.1')): new UnreleasedVersionInfo(v('8.16.1'), '8.16', ':distribution:bwc:bugfix'), + (v('8.17.0')): new UnreleasedVersionInfo(v('8.17.0'), '8.17', ':distribution:bwc:staged'), + (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.x', ':distribution:bwc:minor'), + (v('9.0.0')): new UnreleasedVersionInfo(v('9.0.0'), 'main', ':distribution'), + ] + bwc.wireCompatible == [v('8.18.0'), v('9.0.0')] + bwc.indexCompatible == [v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0'), v('9.0.0')] + } + + def "current version is first new minor in major series"() { + given: + addVersion('7.17.10', '8.9.0') + addVersion('8.16.0', '9.10.0') + addVersion('8.16.1', '9.10.0') + addVersion('8.17.0', '9.10.0') + addVersion('8.18.0', '9.10.0') + addVersion('9.0.0', '10.0.0') + addVersion('9.1.0', '10.0.0') + + when: + def bwc = new BwcVersions(versionLines, v('9.1.0'), ['main', '9.0', '8.18']) + def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } + + then: + unreleased == [ + (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.18', ':distribution:bwc:maintenance'), (v('9.0.0')): new UnreleasedVersionInfo(v('9.0.0'), '9.0', ':distribution:bwc:staged'), - (v('9.1.0')): new UnreleasedVersionInfo(v('9.1.0'), 'main', ':distribution') + (v('9.1.0')): new UnreleasedVersionInfo(v('9.1.0'), 'main', ':distribution'), ] - bwc.wireCompatible == [v('8.17.0'), v('8.17.1'), v('9.0.0'), v('9.1.0')] - bwc.indexCompatible == [v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.17.1'), v('9.0.0'), v('9.1.0')] + bwc.wireCompatible == [v('8.18.0'), v('9.0.0'), v('9.1.0')] + bwc.indexCompatible == [v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0'), v('9.0.0'), v('9.1.0')] } - def "current version is staged major"() { + def "current version is new minor with single bugfix"() { given: - addVersion('8.14.0', '9.9.0') - addVersion('8.14.1', '9.9.0') - addVersion('8.14.2', '9.9.0') - addVersion('8.15.0', '9.9.0') - addVersion('8.15.1', '9.9.0') - addVersion('8.15.2', '9.9.0') + addVersion('7.17.10', '8.9.0') addVersion('8.16.0', '9.10.0') addVersion('8.16.1', '9.10.0') addVersion('8.17.0', '9.10.0') - addVersion('8.17.1', '9.10.0') + addVersion('8.18.0', '9.10.0') addVersion('9.0.0', '10.0.0') + addVersion('9.0.1', '10.0.0') + addVersion('9.1.0', '10.0.0') when: - def bwc = new BwcVersions(versionLines, v('9.0.0')) + def bwc = new BwcVersions(versionLines, v('9.1.0'), ['main', '9.0', '8.18']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ - (v('8.17.1')): new UnreleasedVersionInfo(v('8.17.1'), '8.17', ':distribution:bwc:bugfix'), - (v('9.0.0')): new UnreleasedVersionInfo(v('9.0.0'), 'main', ':distribution'), + (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.18', ':distribution:bwc:maintenance'), + (v('9.0.1')): new UnreleasedVersionInfo(v('9.0.1'), '9.0', ':distribution:bwc:bugfix'), + (v('9.1.0')): new UnreleasedVersionInfo(v('9.1.0'), 'main', ':distribution'), ] - bwc.wireCompatible == [v('8.17.0'), v('8.17.1'), v('9.0.0')] - bwc.indexCompatible == [v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.17.1'), v('9.0.0')] + bwc.wireCompatible == [v('8.18.0'), v('9.0.0'), v('9.0.1'), v('9.1.0')] + bwc.indexCompatible == [v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0'), v('9.0.0'), v('9.0.1'), v('9.1.0')] } - def "current version is major with unreleased next minor"() { + def "current version is new minor with single bugfix and staged minor"() { given: + addVersion('7.17.10', '8.9.0') + addVersion('8.16.0', '9.10.0') + addVersion('8.16.1', '9.10.0') + addVersion('8.17.0', '9.10.0') + addVersion('8.18.0', '9.10.0') + addVersion('9.0.0', '10.0.0') + addVersion('9.0.1', '10.0.0') + addVersion('9.1.0', '10.0.0') + addVersion('9.2.0', '10.0.0') + + when: + def bwc = new BwcVersions(versionLines, v('9.2.0'), ['main', '9.1', '9.0', '8.18']) + def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } + + then: + unreleased == [ + (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.18', ':distribution:bwc:maintenance'), + (v('9.0.1')): new UnreleasedVersionInfo(v('9.0.1'), '9.0', ':distribution:bwc:bugfix'), + (v('9.1.0')): new UnreleasedVersionInfo(v('9.1.0'), '9.1', ':distribution:bwc:staged'), + (v('9.2.0')): new UnreleasedVersionInfo(v('9.2.0'), 'main', ':distribution'), + ] + bwc.wireCompatible == [v('8.18.0'), v('9.0.0'), v('9.0.1'), v('9.1.0'), v('9.2.0')] + bwc.indexCompatible == [v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0'), v('9.0.0'), v('9.0.1'), v('9.1.0'), v('9.2.0')] + } + + def "current version is next minor"() { + given: + addVersion('7.16.3', '8.9.0') + addVersion('7.17.0', '8.9.0') + addVersion('7.17.1', '8.9.0') addVersion('8.14.0', '9.9.0') addVersion('8.14.1', '9.9.0') addVersion('8.14.2', '9.9.0') @@ -113,24 +167,29 @@ class BwcVersionsSpec extends Specification { addVersion('8.16.0', '9.10.0') addVersion('8.16.1', '9.10.0') addVersion('8.17.0', '9.10.0') - addVersion('9.0.0', '10.0.0') + addVersion('8.17.1', '9.10.0') + addVersion('8.18.0', '9.10.0') when: - def bwc = new BwcVersions(versionLines, v('9.0.0')) + def bwc = new BwcVersions(versionLines, v('8.18.0'), ['main', '8.x', '8.17', '8.16', '7.17']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ - (v('8.16.1')): new UnreleasedVersionInfo(v('8.16.1'), '8.16', ':distribution:bwc:bugfix'), - (v('8.17.0')): new UnreleasedVersionInfo(v('8.17.0'), '8.x', ':distribution:bwc:minor'), - (v('9.0.0')): new UnreleasedVersionInfo(v('9.0.0'), 'main', ':distribution'), + (v('7.17.1')): new UnreleasedVersionInfo(v('7.17.1'), '7.17', ':distribution:bwc:maintenance'), + (v('8.16.1')): new UnreleasedVersionInfo(v('8.16.1'), '8.16', ':distribution:bwc:bugfix2'), + (v('8.17.1')): new UnreleasedVersionInfo(v('8.17.1'), '8.17', ':distribution:bwc:bugfix'), + (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.x', ':distribution'), ] - bwc.wireCompatible == [v('8.17.0'), v('9.0.0')] - bwc.indexCompatible == [v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('9.0.0')] + bwc.wireCompatible == [v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.17.1'), v('8.18.0')] + bwc.indexCompatible == [v('7.16.3'), v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.17.1'), v('8.18.0')] } - def "current version is major with staged next minor"() { + def "current version is new minor with staged minor"() { given: + addVersion('7.16.3', '8.9.0') + addVersion('7.17.0', '8.9.0') + addVersion('7.17.1', '8.9.0') addVersion('8.14.0', '9.9.0') addVersion('8.14.1', '9.9.0') addVersion('8.14.2', '9.9.0') @@ -138,26 +197,31 @@ class BwcVersionsSpec extends Specification { addVersion('8.15.1', '9.9.0') addVersion('8.15.2', '9.9.0') addVersion('8.16.0', '9.10.0') + addVersion('8.16.1', '9.10.0') addVersion('8.17.0', '9.10.0') - addVersion('9.0.0', '10.0.0') + addVersion('8.18.0', '9.10.0') when: - def bwc = new BwcVersions(versionLines, v('9.0.0')) + def bwc = new BwcVersions(versionLines, v('8.18.0'), ['main', '8.x', '8.17', '8.16', '8.15', '7.17']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ - (v('8.15.2')): new UnreleasedVersionInfo(v('8.15.2'), '8.15', ':distribution:bwc:bugfix'), - (v('8.16.0')): new UnreleasedVersionInfo(v('8.16.0'), '8.16', ':distribution:bwc:staged'), - (v('8.17.0')): new UnreleasedVersionInfo(v('8.17.0'), '8.x', ':distribution:bwc:minor'), - (v('9.0.0')): new UnreleasedVersionInfo(v('9.0.0'), 'main', ':distribution'), + (v('7.17.1')): new UnreleasedVersionInfo(v('7.17.1'), '7.17', ':distribution:bwc:maintenance'), + (v('8.15.2')): new UnreleasedVersionInfo(v('8.15.2'), '8.15', ':distribution:bwc:bugfix2'), + (v('8.16.1')): new UnreleasedVersionInfo(v('8.16.1'), '8.16', ':distribution:bwc:bugfix'), + (v('8.17.0')): new UnreleasedVersionInfo(v('8.17.0'), '8.17', ':distribution:bwc:staged'), + (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.x', ':distribution'), ] - bwc.wireCompatible == [v('8.17.0'), v('9.0.0')] - bwc.indexCompatible == [v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.17.0'), v('9.0.0')] + bwc.wireCompatible == [v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0')] + bwc.indexCompatible == [v('7.16.3'), v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0')] } - def "current version is next bugfix"() { + def "current version is first bugfix"() { given: + addVersion('7.16.3', '8.9.0') + addVersion('7.17.0', '8.9.0') + addVersion('7.17.1', '8.9.0') addVersion('8.14.0', '9.9.0') addVersion('8.14.1', '9.9.0') addVersion('8.14.2', '9.9.0') @@ -166,52 +230,44 @@ class BwcVersionsSpec extends Specification { addVersion('8.15.2', '9.9.0') addVersion('8.16.0', '9.10.0') addVersion('8.16.1', '9.10.0') - addVersion('8.17.0', '9.10.0') - addVersion('8.17.1', '9.10.0') - addVersion('9.0.0', '10.0.0') - addVersion('9.0.1', '10.0.0') when: - def bwc = new BwcVersions(versionLines, v('9.0.1')) + def bwc = new BwcVersions(versionLines, v('8.16.1'), ['main', '8.x', '8.17', '8.16', '8.15', '7.17']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ - (v('8.17.1')): new UnreleasedVersionInfo(v('8.17.1'), '8.17', ':distribution:bwc:maintenance'), - (v('9.0.1')): new UnreleasedVersionInfo(v('9.0.1'), 'main', ':distribution'), + (v('7.17.1')): new UnreleasedVersionInfo(v('7.17.1'), '7.17', ':distribution:bwc:maintenance'), + (v('8.15.2')): new UnreleasedVersionInfo(v('8.15.2'), '8.15', ':distribution:bwc:bugfix'), + (v('8.16.1')): new UnreleasedVersionInfo(v('8.16.1'), '8.16', ':distribution'), ] - bwc.wireCompatible == [v('8.17.0'), v('8.17.1'), v('9.0.0'), v('9.0.1')] - bwc.indexCompatible == [v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.17.1'), v('9.0.0'), v('9.0.1')] + bwc.wireCompatible == [v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1')] + bwc.indexCompatible == [v('7.16.3'), v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1')] } - def "current version is next minor with no staged releases"() { + def "current version is second bugfix"() { given: + addVersion('7.16.3', '8.9.0') + addVersion('7.17.0', '8.9.0') + addVersion('7.17.1', '8.9.0') addVersion('8.14.0', '9.9.0') addVersion('8.14.1', '9.9.0') addVersion('8.14.2', '9.9.0') addVersion('8.15.0', '9.9.0') addVersion('8.15.1', '9.9.0') addVersion('8.15.2', '9.9.0') - addVersion('8.16.0', '9.10.0') - addVersion('8.16.1', '9.10.0') - addVersion('8.17.0', '9.10.0') - addVersion('8.17.1', '9.10.0') - addVersion('9.0.0', '10.0.0') - addVersion('9.0.1', '10.0.0') - addVersion('9.1.0', '10.1.0') when: - def bwc = new BwcVersions(versionLines, v('9.1.0')) + def bwc = new BwcVersions(versionLines, v('8.15.2'), ['main', '8.x', '8.17', '8.16', '8.15', '7.17']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ - (v('8.17.1')): new UnreleasedVersionInfo(v('8.17.1'), '8.17', ':distribution:bwc:maintenance'), - (v('9.0.1')): new UnreleasedVersionInfo(v('9.0.1'), '9.0', ':distribution:bwc:bugfix'), - (v('9.1.0')): new UnreleasedVersionInfo(v('9.1.0'), 'main', ':distribution') + (v('7.17.1')): new UnreleasedVersionInfo(v('7.17.1'), '7.17', ':distribution:bwc:maintenance'), + (v('8.15.2')): new UnreleasedVersionInfo(v('8.15.2'), '8.15', ':distribution'), ] - bwc.wireCompatible == [v('8.17.0'), v('8.17.1'), v('9.0.0'), v('9.0.1'), v('9.1.0')] - bwc.indexCompatible == [v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.17.1'), v('9.0.0'), v('9.0.1'), v('9.1.0')] + bwc.wireCompatible == [v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2')] + bwc.indexCompatible == [v('7.16.3'), v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2')] } private void addVersion(String elasticsearch, String lucene) { diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/AbstractDistributionDownloadPluginTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/AbstractDistributionDownloadPluginTests.java index 639dec280ae9a..7512fa20814c6 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/AbstractDistributionDownloadPluginTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/AbstractDistributionDownloadPluginTests.java @@ -16,6 +16,7 @@ import java.io.File; import java.util.Arrays; +import java.util.List; public class AbstractDistributionDownloadPluginTests { protected static Project rootProject; @@ -28,22 +29,27 @@ public class AbstractDistributionDownloadPluginTests { protected static final Version BWC_STAGED_VERSION = Version.fromString("1.0.0"); protected static final Version BWC_BUGFIX_VERSION = Version.fromString("1.0.1"); protected static final Version BWC_MAINTENANCE_VERSION = Version.fromString("0.90.1"); + protected static final List DEVELOPMENT_BRANCHES = Arrays.asList("main", "1.1", "1.0", "0.90"); protected static final BwcVersions BWC_MINOR = new BwcVersions( BWC_MAJOR_VERSION, - Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION) + Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION), + DEVELOPMENT_BRANCHES ); protected static final BwcVersions BWC_STAGED = new BwcVersions( BWC_MAJOR_VERSION, - Arrays.asList(BWC_MAINTENANCE_VERSION, BWC_STAGED_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION) + Arrays.asList(BWC_MAINTENANCE_VERSION, BWC_STAGED_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION), + DEVELOPMENT_BRANCHES ); protected static final BwcVersions BWC_BUGFIX = new BwcVersions( BWC_MAJOR_VERSION, - Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION) + Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION), + DEVELOPMENT_BRANCHES ); protected static final BwcVersions BWC_MAINTENANCE = new BwcVersions( BWC_MINOR_VERSION, - Arrays.asList(BWC_MAINTENANCE_VERSION, BWC_BUGFIX_VERSION, BWC_MINOR_VERSION) + Arrays.asList(BWC_MAINTENANCE_VERSION, BWC_BUGFIX_VERSION, BWC_MINOR_VERSION), + DEVELOPMENT_BRANCHES ); protected static String projectName(String base, boolean bundledJdk) { diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index f3f8e4703eba2..07214b5fbf845 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -156,12 +156,12 @@ abstract class AbstractGradleFuncTest extends Specification { File internalBuild( List extraPlugins = [], - String bugfix = "7.15.2", - String bugfixLucene = "8.9.0", - String staged = "7.16.0", - String stagedLucene = "8.10.0", - String minor = "8.0.0", - String minorLucene = "9.0.0" + String maintenance = "7.16.10", + String bugfix2 = "8.1.3", + String bugfix = "8.2.1", + String staged = "8.3.0", + String minor = "8.4.0", + String current = "9.0.0" ) { buildFile << """plugins { id 'elasticsearch.global-build-info' @@ -172,15 +172,17 @@ abstract class AbstractGradleFuncTest extends Specification { import org.elasticsearch.gradle.internal.BwcVersions import org.elasticsearch.gradle.Version - Version currentVersion = Version.fromString("8.1.0") + Version currentVersion = Version.fromString("${current}") def versionList = [ + Version.fromString("$maintenance"), + Version.fromString("$bugfix2"), Version.fromString("$bugfix"), Version.fromString("$staged"), Version.fromString("$minor"), currentVersion ] - BwcVersions versions = new BwcVersions(currentVersion, versionList) + BwcVersions versions = new BwcVersions(currentVersion, versionList, ['main', '8.x', '8.3', '8.2', '8.1', '7.16']) buildParams.getBwcVersionsProperty().set(versions) """ } diff --git a/distribution/bwc/bugfix2/build.gradle b/distribution/bwc/bugfix2/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/settings.gradle b/settings.gradle index 4722fc311480a..747fbb3e439fe 100644 --- a/settings.gradle +++ b/settings.gradle @@ -73,6 +73,7 @@ List projects = [ 'distribution:packages:aarch64-rpm', 'distribution:packages:rpm', 'distribution:bwc:bugfix', + 'distribution:bwc:bugfix2', 'distribution:bwc:maintenance', 'distribution:bwc:minor', 'distribution:bwc:staged', From 584918e39d5f436a20f010163a3ae44fa99046ca Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 5 Dec 2024 20:27:42 +0100 Subject: [PATCH 058/204] Save duplicate REST client in ESRestTestCase (#117910) I debugged some tests today and noticed that these two clients are the same in almost all cases, no need to use extra connections. Might give usa small speedup for these tests that tend to be quite slow relative to the node client based tests. --- .../java/org/elasticsearch/test/rest/ESRestTestCase.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index b4f4243fb90fd..4428afaaeabe5 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -333,8 +333,11 @@ public void initClient() throws IOException { assert testFeatureServiceInitialized() == false; clusterHosts = parseClusterHosts(getTestRestCluster()); logger.info("initializing REST clients against {}", clusterHosts); - client = buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[clusterHosts.size()])); - adminClient = buildClient(restAdminSettings(), clusterHosts.toArray(new HttpHost[clusterHosts.size()])); + var clientSettings = restClientSettings(); + var adminSettings = restAdminSettings(); + var hosts = clusterHosts.toArray(new HttpHost[0]); + client = buildClient(clientSettings, hosts); + adminClient = clientSettings.equals(adminSettings) ? client : buildClient(adminSettings, hosts); availableFeatures = EnumSet.of(ProductFeature.LEGACY_TEMPLATES); Set versions = new HashSet<>(); From fd81c5111878d4cdbbf299976377a3fffd41cb29 Mon Sep 17 00:00:00 2001 From: Sam Xiao Date: Thu, 5 Dec 2024 14:32:25 -0500 Subject: [PATCH 059/204] Unmute BWC tests FullClusterRestartIT (#118038) --- muted-tests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index a09e46415fdc1..ee5e3dd42236d 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -2,12 +2,6 @@ tests: - class: "org.elasticsearch.client.RestClientSingleHostIntegTests" issue: "https://github.com/elastic/elasticsearch/issues/102717" method: "testRequestResetAndAbort" -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testSingleDoc {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/111434 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testDataStreams {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/111448 - class: org.elasticsearch.smoketest.WatcherYamlRestIT method: test {p0=watcher/usage/10_basic/Test watcher usage stats output} issue: https://github.com/elastic/elasticsearch/issues/112189 From 62d94f2920d4e315bcd2867b791022e8a4c33b9f Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Thu, 5 Dec 2024 13:41:43 -0800 Subject: [PATCH 060/204] Remove released vs unreleased distinction from VersionUtils (#118108) --- .../java/org/elasticsearch/VersionTests.java | 41 +-- test/framework/build.gradle | 1 - .../org/elasticsearch/test/VersionUtils.java | 139 +-------- .../elasticsearch/test/VersionUtilsTests.java | 273 ++---------------- 4 files changed, 30 insertions(+), 424 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/VersionTests.java b/server/src/test/java/org/elasticsearch/VersionTests.java index 0b35a3cc23c16..5e10a7d37aea1 100644 --- a/server/src/test/java/org/elasticsearch/VersionTests.java +++ b/server/src/test/java/org/elasticsearch/VersionTests.java @@ -179,8 +179,7 @@ public void testParseVersion() { } public void testAllVersionsMatchId() throws Exception { - final Set releasedVersions = new HashSet<>(VersionUtils.allReleasedVersions()); - final Set unreleasedVersions = new HashSet<>(VersionUtils.allUnreleasedVersions()); + final Set versions = new HashSet<>(VersionUtils.allVersions()); Map maxBranchVersions = new HashMap<>(); for (java.lang.reflect.Field field : Version.class.getFields()) { if (field.getName().matches("_ID")) { @@ -195,43 +194,15 @@ public void testAllVersionsMatchId() throws Exception { Version v = (Version) versionConstant.get(null); logger.debug("Checking {}", v); - if (field.getName().endsWith("_UNRELEASED")) { - assertTrue(unreleasedVersions.contains(v)); - } else { - assertTrue(releasedVersions.contains(v)); - } + assertTrue(versions.contains(v)); assertEquals("Version id " + field.getName() + " does not point to " + constantName, v, Version.fromId(versionId)); assertEquals("Version " + constantName + " does not have correct id", versionId, v.id); String number = v.toString(); assertEquals("V_" + number.replace('.', '_'), constantName); - - // only the latest version for a branch should be a snapshot (ie unreleased) - String branchName = "" + v.major + "." + v.minor; - Version maxBranchVersion = maxBranchVersions.get(branchName); - if (maxBranchVersion == null) { - maxBranchVersions.put(branchName, v); - } else if (v.after(maxBranchVersion)) { - if (v == Version.CURRENT) { - // Current is weird - it counts as released even though it shouldn't. - continue; - } - assertFalse( - "Version " + maxBranchVersion + " cannot be a snapshot because version " + v + " exists", - VersionUtils.allUnreleasedVersions().contains(maxBranchVersion) - ); - maxBranchVersions.put(branchName, v); - } } } } - public static void assertUnknownVersion(Version version) { - assertFalse( - "Version " + version + " has been releaed don't use a new instance of this version", - VersionUtils.allReleasedVersions().contains(version) - ); - } - public void testIsCompatible() { assertTrue(isCompatible(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion())); assertFalse(isCompatible(Version.V_7_0_0, Version.V_8_0_0)); @@ -279,14 +250,6 @@ public boolean isCompatible(Version left, Version right) { return result; } - // This exists because 5.1.0 was never released due to a mistake in the release process. - // This verifies that we never declare the version as "released" accidentally. - // It would never pass qa tests later on, but those come very far in the build and this is quick to check now. - public void testUnreleasedVersion() { - Version VERSION_5_1_0_UNRELEASED = Version.fromString("5.1.0"); - VersionTests.assertUnknownVersion(VERSION_5_1_0_UNRELEASED); - } - public void testIllegalMinorAndPatchNumbers() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> Version.fromString("8.2.999")); assertThat( diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 126b95041da11..c7e08eb3cdfa9 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -86,7 +86,6 @@ tasks.named("thirdPartyAudit").configure { tasks.named("test").configure { systemProperty 'tests.gradle_index_compat_versions', buildParams.bwcVersions.indexCompatible.join(',') systemProperty 'tests.gradle_wire_compat_versions', buildParams.bwcVersions.wireCompatible.join(',') - systemProperty 'tests.gradle_unreleased_versions', buildParams.bwcVersions.unreleased.join(',') } tasks.register("integTest", Test) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java index d561c5512b614..8b7ab620774b9 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java @@ -12,132 +12,15 @@ import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Tuple; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Random; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** Utilities for selecting versions in tests */ public class VersionUtils { - /** - * Sort versions that have backwards compatibility guarantees from - * those that don't. Doesn't actually check whether or not the versions - * are released, instead it relies on gradle to have already checked - * this which it does in {@code :core:verifyVersions}. So long as the - * rules here match up with the rules in gradle then this should - * produce sensible results. - * @return a tuple containing versions with backwards compatibility - * guarantees in v1 and versions without the guranteees in v2 - */ - static Tuple, List> resolveReleasedVersions(Version current, Class versionClass) { - // group versions into major version - Map> majorVersions = Version.getDeclaredVersions(versionClass) - .stream() - .collect(Collectors.groupingBy(v -> (int) v.major)); - // this breaks b/c 5.x is still in version list but master doesn't care about it! - // assert majorVersions.size() == 2; - // TODO: remove oldVersions, we should only ever have 2 majors in Version - List> oldVersions = splitByMinor(majorVersions.getOrDefault((int) current.major - 2, Collections.emptyList())); - List> previousMajor = splitByMinor(majorVersions.get((int) current.major - 1)); - List> currentMajor = splitByMinor(majorVersions.get((int) current.major)); - - List unreleasedVersions = new ArrayList<>(); - final List> stableVersions; - if (currentMajor.size() == 1) { - // on master branch - stableVersions = previousMajor; - // remove current - moveLastToUnreleased(currentMajor, unreleasedVersions); - } else { - // on a stable or release branch, ie N.x - stableVersions = currentMajor; - // remove the next maintenance bugfix - moveLastToUnreleased(previousMajor, unreleasedVersions); - } - - // remove next minor - Version lastMinor = moveLastToUnreleased(stableVersions, unreleasedVersions); - if (lastMinor.revision == 0) { - if (stableVersions.get(stableVersions.size() - 1).size() == 1) { - // a minor is being staged, which is also unreleased - moveLastToUnreleased(stableVersions, unreleasedVersions); - } - // remove the next bugfix - if (stableVersions.isEmpty() == false) { - moveLastToUnreleased(stableVersions, unreleasedVersions); - } - } - - // If none of the previous major was released, then the last minor and bugfix of the old version was not released either. - if (previousMajor.isEmpty()) { - assert currentMajor.isEmpty() : currentMajor; - // minor of the old version is being staged - moveLastToUnreleased(oldVersions, unreleasedVersions); - // bugix of the old version is also being staged - moveLastToUnreleased(oldVersions, unreleasedVersions); - } - List releasedVersions = Stream.of(oldVersions, previousMajor, currentMajor) - .flatMap(List::stream) - .flatMap(List::stream) - .collect(Collectors.toList()); - Collections.sort(unreleasedVersions); // we add unreleased out of order, so need to sort here - return new Tuple<>(Collections.unmodifiableList(releasedVersions), Collections.unmodifiableList(unreleasedVersions)); - } - - // split the given versions into sub lists grouped by minor version - private static List> splitByMinor(List versions) { - Map> byMinor = versions.stream().collect(Collectors.groupingBy(v -> (int) v.minor)); - return byMinor.entrySet().stream().sorted(Map.Entry.comparingByKey()).map(Map.Entry::getValue).collect(Collectors.toList()); - } - - // move the last version of the last minor in versions to the unreleased versions - private static Version moveLastToUnreleased(List> versions, List unreleasedVersions) { - List lastMinor = new ArrayList<>(versions.get(versions.size() - 1)); - Version lastVersion = lastMinor.remove(lastMinor.size() - 1); - if (lastMinor.isEmpty()) { - versions.remove(versions.size() - 1); - } else { - versions.set(versions.size() - 1, lastMinor); - } - unreleasedVersions.add(lastVersion); - return lastVersion; - } - - private static final List RELEASED_VERSIONS; - private static final List UNRELEASED_VERSIONS; - private static final List ALL_VERSIONS; - - static { - Tuple, List> versions = resolveReleasedVersions(Version.CURRENT, Version.class); - RELEASED_VERSIONS = versions.v1(); - UNRELEASED_VERSIONS = versions.v2(); - List allVersions = new ArrayList<>(RELEASED_VERSIONS.size() + UNRELEASED_VERSIONS.size()); - allVersions.addAll(RELEASED_VERSIONS); - allVersions.addAll(UNRELEASED_VERSIONS); - Collections.sort(allVersions); - ALL_VERSIONS = Collections.unmodifiableList(allVersions); - } - - /** - * Returns an immutable, sorted list containing all released versions. - */ - public static List allReleasedVersions() { - return RELEASED_VERSIONS; - } - - /** - * Returns an immutable, sorted list containing all unreleased versions. - */ - public static List allUnreleasedVersions() { - return UNRELEASED_VERSIONS; - } + private static final List ALL_VERSIONS = Version.getDeclaredVersions(Version.class); /** * Returns an immutable, sorted list containing all versions, both released and unreleased. @@ -147,16 +30,16 @@ public static List allVersions() { } /** - * Get the released version before {@code version}. + * Get the version before {@code version}. */ public static Version getPreviousVersion(Version version) { - for (int i = RELEASED_VERSIONS.size() - 1; i >= 0; i--) { - Version v = RELEASED_VERSIONS.get(i); + for (int i = ALL_VERSIONS.size() - 1; i >= 0; i--) { + Version v = ALL_VERSIONS.get(i); if (v.before(version)) { return v; } } - throw new IllegalArgumentException("couldn't find any released versions before [" + version + "]"); + throw new IllegalArgumentException("couldn't find any versions before [" + version + "]"); } /** @@ -169,22 +52,22 @@ public static Version getPreviousVersion() { } /** - * Returns the released {@link Version} before the {@link Version#CURRENT} + * Returns the {@link Version} before the {@link Version#CURRENT} * where the minor version is less than the currents minor version. */ public static Version getPreviousMinorVersion() { - for (int i = RELEASED_VERSIONS.size() - 1; i >= 0; i--) { - Version v = RELEASED_VERSIONS.get(i); + for (int i = ALL_VERSIONS.size() - 1; i >= 0; i--) { + Version v = ALL_VERSIONS.get(i); if (v.minor < Version.CURRENT.minor || v.major < Version.CURRENT.major) { return v; } } - throw new IllegalArgumentException("couldn't find any released versions of the minor before [" + Build.current().version() + "]"); + throw new IllegalArgumentException("couldn't find any versions of the minor before [" + Build.current().version() + "]"); } - /** Returns the oldest released {@link Version} */ + /** Returns the oldest {@link Version} */ public static Version getFirstVersion() { - return RELEASED_VERSIONS.get(0); + return ALL_VERSIONS.get(0); } /** Returns a random {@link Version} from all available versions. */ diff --git a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java index e0013e06f3248..5ae7e5640fc91 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java @@ -9,19 +9,11 @@ package org.elasticsearch.test; import org.elasticsearch.Version; -import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.Tuple; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import static org.elasticsearch.Version.fromId; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.lessThanOrEqualTo; /** * Tests VersionUtils. Note: this test should remain unchanged across major versions @@ -30,7 +22,7 @@ public class VersionUtilsTests extends ESTestCase { public void testAllVersionsSorted() { - List allVersions = VersionUtils.allReleasedVersions(); + List allVersions = VersionUtils.allVersions(); for (int i = 0, j = 1; j < allVersions.size(); ++i, ++j) { assertTrue(allVersions.get(i).before(allVersions.get(j))); } @@ -58,9 +50,9 @@ public void testRandomVersionBetween() { got = VersionUtils.randomVersionBetween(random(), null, fromId(7000099)); assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); assertTrue(got.onOrBefore(fromId(7000099))); - got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.allReleasedVersions().get(0)); + got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.allVersions().get(0)); assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); - assertTrue(got.onOrBefore(VersionUtils.allReleasedVersions().get(0))); + assertTrue(got.onOrBefore(VersionUtils.allVersions().get(0))); // unbounded upper got = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), null); @@ -83,265 +75,34 @@ public void testRandomVersionBetween() { assertEquals(got, VersionUtils.getFirstVersion()); got = VersionUtils.randomVersionBetween(random(), Version.CURRENT, null); assertEquals(got, Version.CURRENT); - - if (Booleans.parseBoolean(System.getProperty("build.snapshot", "true"))) { - // max or min can be an unreleased version - final Version unreleased = randomFrom(VersionUtils.allUnreleasedVersions()); - assertThat(VersionUtils.randomVersionBetween(random(), null, unreleased), lessThanOrEqualTo(unreleased)); - assertThat(VersionUtils.randomVersionBetween(random(), unreleased, null), greaterThanOrEqualTo(unreleased)); - assertEquals(unreleased, VersionUtils.randomVersionBetween(random(), unreleased, unreleased)); - } - } - - public static class TestReleaseBranch { - public static final Version V_4_0_0 = Version.fromString("4.0.0"); - public static final Version V_4_0_1 = Version.fromString("4.0.1"); - public static final Version V_5_3_0 = Version.fromString("5.3.0"); - public static final Version V_5_3_1 = Version.fromString("5.3.1"); - public static final Version V_5_3_2 = Version.fromString("5.3.2"); - public static final Version V_5_4_0 = Version.fromString("5.4.0"); - public static final Version V_5_4_1 = Version.fromString("5.4.1"); - public static final Version CURRENT = V_5_4_1; - } - - public void testResolveReleasedVersionsForReleaseBranch() { - Tuple, List> t = VersionUtils.resolveReleasedVersions(TestReleaseBranch.CURRENT, TestReleaseBranch.class); - List released = t.v1(); - List unreleased = t.v2(); - - assertThat( - released, - equalTo( - Arrays.asList( - TestReleaseBranch.V_4_0_0, - TestReleaseBranch.V_5_3_0, - TestReleaseBranch.V_5_3_1, - TestReleaseBranch.V_5_3_2, - TestReleaseBranch.V_5_4_0 - ) - ) - ); - assertThat(unreleased, equalTo(Arrays.asList(TestReleaseBranch.V_4_0_1, TestReleaseBranch.V_5_4_1))); - } - - public static class TestStableBranch { - public static final Version V_4_0_0 = Version.fromString("4.0.0"); - public static final Version V_4_0_1 = Version.fromString("4.0.1"); - public static final Version V_5_0_0 = Version.fromString("5.0.0"); - public static final Version V_5_0_1 = Version.fromString("5.0.1"); - public static final Version V_5_0_2 = Version.fromString("5.0.2"); - public static final Version V_5_1_0 = Version.fromString("5.1.0"); - public static final Version CURRENT = V_5_1_0; - } - - public void testResolveReleasedVersionsForUnreleasedStableBranch() { - Tuple, List> t = VersionUtils.resolveReleasedVersions(TestStableBranch.CURRENT, TestStableBranch.class); - List released = t.v1(); - List unreleased = t.v2(); - - assertThat(released, equalTo(Arrays.asList(TestStableBranch.V_4_0_0, TestStableBranch.V_5_0_0, TestStableBranch.V_5_0_1))); - assertThat(unreleased, equalTo(Arrays.asList(TestStableBranch.V_4_0_1, TestStableBranch.V_5_0_2, TestStableBranch.V_5_1_0))); - } - - public static class TestStableBranchBehindStableBranch { - public static final Version V_4_0_0 = Version.fromString("4.0.0"); - public static final Version V_4_0_1 = Version.fromString("4.0.1"); - public static final Version V_5_3_0 = Version.fromString("5.3.0"); - public static final Version V_5_3_1 = Version.fromString("5.3.1"); - public static final Version V_5_3_2 = Version.fromString("5.3.2"); - public static final Version V_5_4_0 = Version.fromString("5.4.0"); - public static final Version V_5_5_0 = Version.fromString("5.5.0"); - public static final Version CURRENT = V_5_5_0; - } - - public void testResolveReleasedVersionsForStableBranchBehindStableBranch() { - Tuple, List> t = VersionUtils.resolveReleasedVersions( - TestStableBranchBehindStableBranch.CURRENT, - TestStableBranchBehindStableBranch.class - ); - List released = t.v1(); - List unreleased = t.v2(); - - assertThat( - released, - equalTo( - Arrays.asList( - TestStableBranchBehindStableBranch.V_4_0_0, - TestStableBranchBehindStableBranch.V_5_3_0, - TestStableBranchBehindStableBranch.V_5_3_1 - ) - ) - ); - assertThat( - unreleased, - equalTo( - Arrays.asList( - TestStableBranchBehindStableBranch.V_4_0_1, - TestStableBranchBehindStableBranch.V_5_3_2, - TestStableBranchBehindStableBranch.V_5_4_0, - TestStableBranchBehindStableBranch.V_5_5_0 - ) - ) - ); - } - - public static class TestUnstableBranch { - public static final Version V_5_3_0 = Version.fromString("5.3.0"); - public static final Version V_5_3_1 = Version.fromString("5.3.1"); - public static final Version V_5_3_2 = Version.fromString("5.3.2"); - public static final Version V_5_4_0 = Version.fromString("5.4.0"); - public static final Version V_6_0_0 = Version.fromString("6.0.0"); - public static final Version CURRENT = V_6_0_0; - } - - public void testResolveReleasedVersionsForUnstableBranch() { - Tuple, List> t = VersionUtils.resolveReleasedVersions(TestUnstableBranch.CURRENT, TestUnstableBranch.class); - List released = t.v1(); - List unreleased = t.v2(); - - assertThat(released, equalTo(Arrays.asList(TestUnstableBranch.V_5_3_0, TestUnstableBranch.V_5_3_1))); - assertThat(unreleased, equalTo(Arrays.asList(TestUnstableBranch.V_5_3_2, TestUnstableBranch.V_5_4_0, TestUnstableBranch.V_6_0_0))); - } - - public static class TestNewMajorRelease { - public static final Version V_5_6_0 = Version.fromString("5.6.0"); - public static final Version V_5_6_1 = Version.fromString("5.6.1"); - public static final Version V_5_6_2 = Version.fromString("5.6.2"); - public static final Version V_6_0_0 = Version.fromString("6.0.0"); - public static final Version V_6_0_1 = Version.fromString("6.0.1"); - public static final Version CURRENT = V_6_0_1; - } - - public void testResolveReleasedVersionsAtNewMajorRelease() { - Tuple, List> t = VersionUtils.resolveReleasedVersions( - TestNewMajorRelease.CURRENT, - TestNewMajorRelease.class - ); - List released = t.v1(); - List unreleased = t.v2(); - - assertThat(released, equalTo(Arrays.asList(TestNewMajorRelease.V_5_6_0, TestNewMajorRelease.V_5_6_1, TestNewMajorRelease.V_6_0_0))); - assertThat(unreleased, equalTo(Arrays.asList(TestNewMajorRelease.V_5_6_2, TestNewMajorRelease.V_6_0_1))); - } - - public static class TestVersionBumpIn6x { - public static final Version V_5_6_0 = Version.fromString("5.6.0"); - public static final Version V_5_6_1 = Version.fromString("5.6.1"); - public static final Version V_5_6_2 = Version.fromString("5.6.2"); - public static final Version V_6_0_0 = Version.fromString("6.0.0"); - public static final Version V_6_0_1 = Version.fromString("6.0.1"); - public static final Version V_6_1_0 = Version.fromString("6.1.0"); - public static final Version CURRENT = V_6_1_0; - } - - public void testResolveReleasedVersionsAtVersionBumpIn6x() { - Tuple, List> t = VersionUtils.resolveReleasedVersions( - TestVersionBumpIn6x.CURRENT, - TestVersionBumpIn6x.class - ); - List released = t.v1(); - List unreleased = t.v2(); - - assertThat(released, equalTo(Arrays.asList(TestVersionBumpIn6x.V_5_6_0, TestVersionBumpIn6x.V_5_6_1, TestVersionBumpIn6x.V_6_0_0))); - assertThat( - unreleased, - equalTo(Arrays.asList(TestVersionBumpIn6x.V_5_6_2, TestVersionBumpIn6x.V_6_0_1, TestVersionBumpIn6x.V_6_1_0)) - ); - } - - public static class TestNewMinorBranchIn6x { - public static final Version V_5_6_0 = Version.fromString("5.6.0"); - public static final Version V_5_6_1 = Version.fromString("5.6.1"); - public static final Version V_5_6_2 = Version.fromString("5.6.2"); - public static final Version V_6_0_0 = Version.fromString("6.0.0"); - public static final Version V_6_0_1 = Version.fromString("6.0.1"); - public static final Version V_6_1_0 = Version.fromString("6.1.0"); - public static final Version V_6_1_1 = Version.fromString("6.1.1"); - public static final Version V_6_1_2 = Version.fromString("6.1.2"); - public static final Version V_6_2_0 = Version.fromString("6.2.0"); - public static final Version CURRENT = V_6_2_0; - } - - public void testResolveReleasedVersionsAtNewMinorBranchIn6x() { - Tuple, List> t = VersionUtils.resolveReleasedVersions( - TestNewMinorBranchIn6x.CURRENT, - TestNewMinorBranchIn6x.class - ); - List released = t.v1(); - List unreleased = t.v2(); - - assertThat( - released, - equalTo( - Arrays.asList( - TestNewMinorBranchIn6x.V_5_6_0, - TestNewMinorBranchIn6x.V_5_6_1, - TestNewMinorBranchIn6x.V_6_0_0, - TestNewMinorBranchIn6x.V_6_0_1, - TestNewMinorBranchIn6x.V_6_1_0, - TestNewMinorBranchIn6x.V_6_1_1 - ) - ) - ); - assertThat( - unreleased, - equalTo(Arrays.asList(TestNewMinorBranchIn6x.V_5_6_2, TestNewMinorBranchIn6x.V_6_1_2, TestNewMinorBranchIn6x.V_6_2_0)) - ); } /** - * Tests that {@link Version#minimumCompatibilityVersion()} and {@link VersionUtils#allReleasedVersions()} + * Tests that {@link Version#minimumCompatibilityVersion()} and {@link VersionUtils#allVersions()} * agree with the list of wire compatible versions we build in gradle. */ public void testGradleVersionsMatchVersionUtils() { // First check the index compatible versions - List released = VersionUtils.allReleasedVersions() + List versions = VersionUtils.allVersions() .stream() /* Java lists all versions from the 5.x series onwards, but we only want to consider * ones that we're supposed to be compatible with. */ .filter(v -> v.onOrAfter(Version.CURRENT.minimumCompatibilityVersion())) + .map(Version::toString) .toList(); - VersionsFromProperty wireCompatible = new VersionsFromProperty("tests.gradle_wire_compat_versions"); - - Version minimumCompatibleVersion = Version.CURRENT.minimumCompatibilityVersion(); - List releasedWireCompatible = released.stream() - .filter(v -> Version.CURRENT.equals(v) == false) - .filter(v -> v.onOrAfter(minimumCompatibleVersion)) - .map(Object::toString) - .toList(); - assertEquals(releasedWireCompatible, wireCompatible.released); - - List unreleasedWireCompatible = VersionUtils.allUnreleasedVersions() - .stream() - .filter(v -> v.onOrAfter(minimumCompatibleVersion)) - .map(Object::toString) - .toList(); - assertEquals(unreleasedWireCompatible, wireCompatible.unreleased); + List gradleVersions = versionFromProperty("tests.gradle_wire_compat_versions"); + assertEquals(versions, gradleVersions); } - /** - * Read a versions system property as set by gradle into a tuple of {@code (releasedVersion, unreleasedVersion)}. - */ - private class VersionsFromProperty { - private final List released = new ArrayList<>(); - private final List unreleased = new ArrayList<>(); - - private VersionsFromProperty(String property) { - Set allUnreleased = new HashSet<>(Arrays.asList(System.getProperty("tests.gradle_unreleased_versions", "").split(","))); - if (allUnreleased.isEmpty()) { - fail("[tests.gradle_unreleased_versions] not set or empty. Gradle should set this before running."); - } - String versions = System.getProperty(property); - assertNotNull("Couldn't find [" + property + "]. Gradle should set this before running the tests.", versions); - logger.info("Looked up versions [{}={}]", property, versions); - - for (String version : versions.split(",")) { - if (allUnreleased.contains(version)) { - unreleased.add(version); - } else { - released.add(version); - } - } + private List versionFromProperty(String property) { + List versions = new ArrayList<>(); + String versionsString = System.getProperty(property); + assertNotNull("Couldn't find [" + property + "]. Gradle should set this before running the tests.", versionsString); + logger.info("Looked up versions [{}={}]", property, versionsString); + for (String version : versionsString.split(",")) { + versions.add(version); } + + return versions; } } From 6adb3e04ba41657824b71d00b66cbefff68e761f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:30:14 +1100 Subject: [PATCH 061/204] Mute org.elasticsearch.packaging.test.ConfigurationTests test30SymlinkedDataPath #118111 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index ee5e3dd42236d..772b03171b237 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -247,6 +247,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/118029 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT issue: https://github.com/elastic/elasticsearch/issues/117981 +- class: org.elasticsearch.packaging.test.ConfigurationTests + method: test30SymlinkedDataPath + issue: https://github.com/elastic/elasticsearch/issues/118111 # Examples: # From 3a292e982f3f9a7e66ecbe8f7892d44cb90533dd Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 5 Dec 2024 15:53:17 -0800 Subject: [PATCH 062/204] Add a diagram of how entitlements loading works (#117513) This commit adds a diagram to source control that explains how the entitlements lib is loaded. --- libs/entitlement/entitlements-loading.svg | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 libs/entitlement/entitlements-loading.svg diff --git a/libs/entitlement/entitlements-loading.svg b/libs/entitlement/entitlements-loading.svg new file mode 100644 index 0000000000000..4f0213b853bee --- /dev/null +++ b/libs/entitlement/entitlements-loading.svg @@ -0,0 +1,4 @@ + + + +
ES main
ES main
Boot Loader
Boot Loader
Platform Loader
Platform Loader
System Loader
System Loader
reflection
reflection
Agent Jar
Agent Jar
Server
Server
(Instrumented)
JDK classes
(Instrumented)...
agent main
(in unnamed module)
agent main...
entitlements ready
entitlements ready
reflection
reflection
Bridge
(patched into java.base)
Bridge...
Entitlements
Entitlements
Entitlements bootstrap
Entitlements bootstrap
  • Grant access to unnamed module
  • Set (static, protected) init arguments
  • Load agent
Grant access to unnamed modu...
(reflectively) call 
entitlements init
with Instrumentation
(reflectively) call...
Entitlements init
Entitlements init
  • Load plugin policies
  • Load server policy
  • Create entitlements manager
    • Policies
    • Method to lookup plugin by Module
  • Set entitlements manager in static (accessible by bridge)
  • Instrument jdk classes
  • run self test (force bridge to capture entitlements manager)
Load plugin policiesLoad server policyCreate e...Text is not SVG - cannot display \ No newline at end of file From e67856895b5a11837ec6c759364177ea23b9000e Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Fri, 6 Dec 2024 14:22:59 +1100 Subject: [PATCH 063/204] Add link to Tasks experimental issue (#118117) --- docs/internal/DistributedArchitectureGuide.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/internal/DistributedArchitectureGuide.md b/docs/internal/DistributedArchitectureGuide.md index 793d38e3d73b3..11a2c860eb326 100644 --- a/docs/internal/DistributedArchitectureGuide.md +++ b/docs/internal/DistributedArchitectureGuide.md @@ -386,6 +386,9 @@ The tasks infrastructure is used to track currently executing operations in the Each individual task is local to a node, but can be related to other tasks, on the same node or other nodes, via a parent-child relationship. +> [!NOTE] +> The Task management API is experimental/beta, its status and outstanding issues can be tracked [here](https://github.com/elastic/elasticsearch/issues/51628). + ### Task tracking and registration Tasks are tracked in-memory on each node in the node's [TaskManager], new tasks are registered via one of the [TaskManager#register] methods. From dc443524e4397d1514d73f13b676d090d40065be Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:42:52 +1100 Subject: [PATCH 064/204] Mute org.elasticsearch.datastreams.ResolveClusterDataStreamIT testClusterResolveWithDataStreamsUsingAlias #118124 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 772b03171b237..99d1e170a4f4e 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -250,6 +250,9 @@ tests: - class: org.elasticsearch.packaging.test.ConfigurationTests method: test30SymlinkedDataPath issue: https://github.com/elastic/elasticsearch/issues/118111 +- class: org.elasticsearch.datastreams.ResolveClusterDataStreamIT + method: testClusterResolveWithDataStreamsUsingAlias + issue: https://github.com/elastic/elasticsearch/issues/118124 # Examples: # From 7bd5c69c8c5ad3c4a21949e80d6bedc14351d073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Fri, 6 Dec 2024 08:16:46 +0100 Subject: [PATCH 065/204] Update ASM 9.7 -> 9.7.1 to support JDK 24 (#118094) --- distribution/tools/plugin-cli/build.gradle | 4 +-- docs/changelog/118094.yaml | 5 +++ gradle/verification-metadata.xml | 31 +++++-------------- libs/entitlement/asm-provider/build.gradle | 4 +-- .../securitymanager-scanner/build.gradle | 4 +-- libs/plugin-scanner/build.gradle | 4 +-- test/immutable-collections-patch/build.gradle | 4 +-- test/logger-usage/build.gradle | 6 ++-- 8 files changed, 26 insertions(+), 36 deletions(-) create mode 100644 docs/changelog/118094.yaml diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index 57750f2162a71..dc2bcd96b8d9f 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -25,8 +25,8 @@ dependencies { implementation project(":libs:plugin-api") implementation project(":libs:plugin-scanner") // TODO: asm is picked up from the plugin scanner, we should consolidate so it is not defined twice - implementation 'org.ow2.asm:asm:9.7' - implementation 'org.ow2.asm:asm-tree:9.7' + implementation 'org.ow2.asm:asm:9.7.1' + implementation 'org.ow2.asm:asm-tree:9.7.1' api "org.bouncycastle:bcpg-fips:1.0.7.1" api "org.bouncycastle:bc-fips:1.0.2.5" diff --git a/docs/changelog/118094.yaml b/docs/changelog/118094.yaml new file mode 100644 index 0000000000000..a8866543fa7d2 --- /dev/null +++ b/docs/changelog/118094.yaml @@ -0,0 +1,5 @@ +pr: 118094 +summary: Update ASM 9.7 -> 9.7.1 to support JDK 24 +area: Infra/Core +type: upgrade +issues: [] diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 37178fd9439d0..9189d2a27f3f3 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -4383,11 +4383,6 @@ - - - - - @@ -4408,9 +4403,9 @@ - - - + + + @@ -4433,11 +4428,6 @@ - - - - - @@ -4478,11 +4468,6 @@ - - - - - @@ -4493,11 +4478,11 @@ - - - - - + + + + + diff --git a/libs/entitlement/asm-provider/build.gradle b/libs/entitlement/asm-provider/build.gradle index 5f968629fe557..dcec0579a5bae 100644 --- a/libs/entitlement/asm-provider/build.gradle +++ b/libs/entitlement/asm-provider/build.gradle @@ -11,10 +11,10 @@ apply plugin: 'elasticsearch.build' dependencies { compileOnly project(':libs:entitlement') - implementation 'org.ow2.asm:asm:9.7' + implementation 'org.ow2.asm:asm:9.7.1' testImplementation project(":test:framework") testImplementation project(":libs:entitlement:bridge") - testImplementation 'org.ow2.asm:asm-util:9.7' + testImplementation 'org.ow2.asm:asm-util:9.7.1' } tasks.named('test').configure { diff --git a/libs/entitlement/tools/securitymanager-scanner/build.gradle b/libs/entitlement/tools/securitymanager-scanner/build.gradle index 8d035c9e847c6..ebb671e5487ef 100644 --- a/libs/entitlement/tools/securitymanager-scanner/build.gradle +++ b/libs/entitlement/tools/securitymanager-scanner/build.gradle @@ -47,8 +47,8 @@ repositories { dependencies { compileOnly(project(':libs:core')) - implementation 'org.ow2.asm:asm:9.7' - implementation 'org.ow2.asm:asm-util:9.7' + implementation 'org.ow2.asm:asm:9.7.1' + implementation 'org.ow2.asm:asm-util:9.7.1' implementation(project(':libs:entitlement:tools:common')) } diff --git a/libs/plugin-scanner/build.gradle b/libs/plugin-scanner/build.gradle index d04af0624b3b1..44e6853140a5b 100644 --- a/libs/plugin-scanner/build.gradle +++ b/libs/plugin-scanner/build.gradle @@ -20,8 +20,8 @@ dependencies { api project(':libs:plugin-api') api project(":libs:x-content") - api 'org.ow2.asm:asm:9.7' - api 'org.ow2.asm:asm-tree:9.7' + api 'org.ow2.asm:asm:9.7.1' + api 'org.ow2.asm:asm-tree:9.7.1' testImplementation "junit:junit:${versions.junit}" testImplementation(project(":test:framework")) { diff --git a/test/immutable-collections-patch/build.gradle b/test/immutable-collections-patch/build.gradle index 85a199af2d477..852a19116fb71 100644 --- a/test/immutable-collections-patch/build.gradle +++ b/test/immutable-collections-patch/build.gradle @@ -17,8 +17,8 @@ configurations { } dependencies { - implementation 'org.ow2.asm:asm:9.7' - implementation 'org.ow2.asm:asm-tree:9.7' + implementation 'org.ow2.asm:asm:9.7.1' + implementation 'org.ow2.asm:asm-tree:9.7.1' } def outputDir = layout.buildDirectory.dir("jdk-patches") diff --git a/test/logger-usage/build.gradle b/test/logger-usage/build.gradle index 8677b1404a727..6d6c5ff889a45 100644 --- a/test/logger-usage/build.gradle +++ b/test/logger-usage/build.gradle @@ -10,9 +10,9 @@ apply plugin: 'elasticsearch.java' dependencies { - api 'org.ow2.asm:asm:9.7' - api 'org.ow2.asm:asm-tree:9.7' - api 'org.ow2.asm:asm-analysis:9.7' + api 'org.ow2.asm:asm:9.7.1' + api 'org.ow2.asm:asm-tree:9.7.1' + api 'org.ow2.asm:asm-analysis:9.7.1' api "org.apache.logging.log4j:log4j-api:${versions.log4j}" testImplementation project(":test:framework") } From 91605860ee9a112815cb20915d07551cefa4d63f Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 6 Dec 2024 08:42:48 +0100 Subject: [PATCH 066/204] Term query for ES|QL (#117359) This commit adds a `term` function for ES|QL to run `TermQueries`. For example: FROM test | WHERE term(content, "dog") --- docs/changelog/117359.yaml | 5 + .../esql/functions/description/term.asciidoc | 5 + .../esql/functions/examples/term.asciidoc | 13 ++ .../functions/kibana/definition/term.json | 85 ++++++++ .../esql/functions/kibana/docs/term.md | 13 ++ .../esql/functions/layout/term.asciidoc | 17 ++ .../esql/functions/parameters/term.asciidoc | 9 + .../esql/functions/signature/term.svg | 1 + .../esql/functions/types/term.asciidoc | 12 + .../src/main/resources/term-function.csv-spec | 206 ++++++++++++++++++ .../xpack/esql/plugin/TermIT.java | 139 ++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 7 +- .../xpack/esql/analysis/Verifier.java | 9 + .../function/EsqlFunctionRegistry.java | 4 +- .../function/fulltext/FullTextWritables.java | 3 + .../expression/function/fulltext/Term.java | 124 +++++++++++ .../physical/local/PushFiltersToSource.java | 3 + .../planner/EsqlExpressionTranslators.java | 10 + .../elasticsearch/xpack/esql/CsvTests.java | 4 + .../xpack/esql/analysis/VerifierTests.java | 54 +++++ .../function/fulltext/TermTests.java | 132 +++++++++++ .../LocalPhysicalPlanOptimizerTests.java | 29 +++ .../rest-api-spec/test/esql/60_usage.yml | 2 +- 23 files changed, 883 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/117359.yaml create mode 100644 docs/reference/esql/functions/description/term.asciidoc create mode 100644 docs/reference/esql/functions/examples/term.asciidoc create mode 100644 docs/reference/esql/functions/kibana/definition/term.json create mode 100644 docs/reference/esql/functions/kibana/docs/term.md create mode 100644 docs/reference/esql/functions/layout/term.asciidoc create mode 100644 docs/reference/esql/functions/parameters/term.asciidoc create mode 100644 docs/reference/esql/functions/signature/term.svg create mode 100644 docs/reference/esql/functions/types/term.asciidoc create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/term-function.csv-spec create mode 100644 x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/TermIT.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/TermTests.java diff --git a/docs/changelog/117359.yaml b/docs/changelog/117359.yaml new file mode 100644 index 0000000000000..87d2d828ace54 --- /dev/null +++ b/docs/changelog/117359.yaml @@ -0,0 +1,5 @@ +pr: 117359 +summary: Term query for ES|QL +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/reference/esql/functions/description/term.asciidoc b/docs/reference/esql/functions/description/term.asciidoc new file mode 100644 index 0000000000000..c43aeb25a0ef7 --- /dev/null +++ b/docs/reference/esql/functions/description/term.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Performs a Term query on the specified field. Returns true if the provided term matches the row. diff --git a/docs/reference/esql/functions/examples/term.asciidoc b/docs/reference/esql/functions/examples/term.asciidoc new file mode 100644 index 0000000000000..b9d57f366294b --- /dev/null +++ b/docs/reference/esql/functions/examples/term.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/term-function.csv-spec[tag=term-with-field] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/term-function.csv-spec[tag=term-with-field-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/term.json b/docs/reference/esql/functions/kibana/definition/term.json new file mode 100644 index 0000000000000..d8bb61fd596a1 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/term.json @@ -0,0 +1,85 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "term", + "description" : "Performs a Term query on the specified field. Returns true if the provided term matches the row.", + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Term you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Term you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Term you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Term you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "from books \n| where term(author, \"gabriel\") \n| keep book_no, title\n| limit 3;" + ], + "preview" : true, + "snapshot_only" : true +} diff --git a/docs/reference/esql/functions/kibana/docs/term.md b/docs/reference/esql/functions/kibana/docs/term.md new file mode 100644 index 0000000000000..83e61a949208d --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/term.md @@ -0,0 +1,13 @@ + + +### TERM +Performs a Term query on the specified field. Returns true if the provided term matches the row. + +``` +from books +| where term(author, "gabriel") +| keep book_no, title +| limit 3; +``` diff --git a/docs/reference/esql/functions/layout/term.asciidoc b/docs/reference/esql/functions/layout/term.asciidoc new file mode 100644 index 0000000000000..1fe94491bed04 --- /dev/null +++ b/docs/reference/esql/functions/layout/term.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-term]] +=== `TERM` + +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/term.svg[Embedded,opts=inline] + +include::../parameters/term.asciidoc[] +include::../description/term.asciidoc[] +include::../types/term.asciidoc[] +include::../examples/term.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/term.asciidoc b/docs/reference/esql/functions/parameters/term.asciidoc new file mode 100644 index 0000000000000..edba8625d04c5 --- /dev/null +++ b/docs/reference/esql/functions/parameters/term.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`field`:: +Field that the query will target. + +`query`:: +Term you wish to find in the provided field. diff --git a/docs/reference/esql/functions/signature/term.svg b/docs/reference/esql/functions/signature/term.svg new file mode 100644 index 0000000000000..955dd7fa215ab --- /dev/null +++ b/docs/reference/esql/functions/signature/term.svg @@ -0,0 +1 @@ +TERM(field,query) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/term.asciidoc b/docs/reference/esql/functions/types/term.asciidoc new file mode 100644 index 0000000000000..7523b29c62b1d --- /dev/null +++ b/docs/reference/esql/functions/types/term.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +field | query | result +keyword | keyword | boolean +keyword | text | boolean +text | keyword | boolean +text | text | boolean +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/term-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/term-function.csv-spec new file mode 100644 index 0000000000000..0c72cad02eed1 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/term-function.csv-spec @@ -0,0 +1,206 @@ +############################################### +# Tests for Term function +# + +termWithTextField +required_capability: term_function + +// tag::term-with-field[] +FROM books +| WHERE TERM(author, "gabriel") +| KEEP book_no, title +| LIMIT 3; +// end::term-with-field[] +ignoreOrder:true + +book_no:keyword | title:text +4814 | El Coronel No Tiene Quien Le Escriba / No One Writes to the Colonel (Spanish Edition) +4917 | Autumn of the Patriarch +6380 | La hojarasca (Spanish Edition) +; + +termWithKeywordField +required_capability: term_function + +from employees +| where term(first_name, "Guoxiang") +| keep emp_no, first_name; + +// tag::term-with-keyword-field-result[] +emp_no:integer | first_name:keyword +10015 | Guoxiang +; +// end::term-with-keyword-field-result[] + +termWithQueryExpressions +required_capability: term_function + +from books +| where term(author, CONCAT("gab", "riel")) +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4814 | El Coronel No Tiene Quien Le Escriba / No One Writes to the Colonel (Spanish Edition) +4917 | Autumn of the Patriarch +6380 | La hojarasca (Spanish Edition) +; + +termAfterKeep +required_capability: term_function + +from books +| keep book_no, author +| where term(author, "faulkner") +| sort book_no +| limit 5; + +book_no:keyword | author:text +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] +2713 | William Faulkner +2847 | Colleen Faulkner +2883 | William Faulkner +3293 | Danny Faulkner +; + +termAfterDrop +required_capability: term_function + +from books +| drop ratings, description, year, publisher, title, author.keyword +| where term(author, "william") +| keep book_no, author +| sort book_no +| limit 2; + +book_no:keyword | author:text +2713 | William Faulkner +2883 | William Faulkner +; + +termAfterEval +required_capability: term_function + +from books +| eval stars = to_long(ratings / 2.0) +| where term(author, "colleen") +| sort book_no +| keep book_no, author, stars +| limit 2; + +book_no:keyword | author:text | stars:long +2847 | Colleen Faulkner | 3 +4502 | Colleen Faulkner | 3 +; + +termWithConjunction +required_capability: term_function + +from books +| where term(author, "tolkien") and ratings > 4.95 +| eval author = mv_sort(author) +| keep book_no, ratings, author; +ignoreOrder:true + +book_no:keyword | ratings:double | author:keyword +2301 | 5.0 | John Ronald Reuel Tolkien +3254 | 5.0 | [Christopher Tolkien, John Ronald Reuel Tolkien] +7350 | 5.0 | [Christopher Tolkien, John Ronald Reuel Tolkien] +; + +termWithConjunctionAndSort +required_capability: term_function + +from books +| where term(author, "tolkien") and ratings > 4.95 +| eval author = mv_sort(author) +| keep book_no, ratings, author +| sort book_no; + +book_no:keyword | ratings:double | author:keyword +2301 | 5.0 | John Ronald Reuel Tolkien +3254 | 5.0 | [Christopher Tolkien, John Ronald Reuel Tolkien] +7350 | 5.0 | [Christopher Tolkien, John Ronald Reuel Tolkien] +; + +termWithFunctionPushedToLucene +required_capability: term_function + +from hosts +| where term(host, "beta") and cidr_match(ip1, "127.0.0.2/32", "127.0.0.3/32") +| keep card, host, ip0, ip1; +ignoreOrder:true + +card:keyword |host:keyword |ip0:ip |ip1:ip +eth1 |beta |127.0.0.1 |127.0.0.2 +; + +termWithNonPushableConjunction +required_capability: term_function + +from books +| where term(title, "rings") and length(title) > 75 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +; + +termWithMultipleWhereClauses +required_capability: term_function + +from books +| where term(title, "rings") +| where term(title, "lord") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2675 | The Lord of the Rings - Boxed Set +2714 | Return of the King Being the Third Part of The Lord of the Rings +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; + +termWithMultivaluedField +required_capability: term_function + +from employees +| where term(job_positions, "Data Scientist") +| keep emp_no, first_name, last_name +| sort emp_no asc +| limit 2; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10014 | Berni | Genin +10017 | Cristinel | Bouloucos +; + +testWithMultiValuedFieldWithConjunction +required_capability: term_function + +from employees +| where term(job_positions, "Data Scientist") and term(first_name, "Cristinel") +| keep emp_no, first_name, last_name +| limit 1; + +emp_no:integer | first_name:keyword | last_name:keyword +10017 | Cristinel | Bouloucos +; + +termWithConjQueryStringFunctions +required_capability: term_function +required_capability: qstr_function + +from employees +| where term(job_positions, "Data Scientist") and qstr("first_name: Cristinel and gender: F") +| keep emp_no, first_name, last_name +| sort emp_no ASC +| limit 1; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10017 | Cristinel | Bouloucos +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/TermIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/TermIT.java new file mode 100644 index 0000000000000..4bb4897c9db5f --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/TermIT.java @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plugin; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; +import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; +import org.junit.Before; + +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.CoreMatchers.containsString; + +public class TermIT extends AbstractEsqlIntegTestCase { + + @Before + public void setupIndex() { + createAndPopulateIndex(); + } + + @Override + protected EsqlQueryResponse run(EsqlQueryRequest request) { + assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); + return super.run(request); + } + + public void testSimpleTermQuery() throws Exception { + var query = """ + FROM test + | WHERE term(content,"dog") + | KEEP id + | SORT id + """; + + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValues(resp.values(), List.of(List.of(1), List.of(3), List.of(4), List.of(5))); + } + } + + public void testTermWithinEval() { + var query = """ + FROM test + | EVAL term_query = term(title,"fox") + """; + + var error = expectThrows(VerificationException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("[Term] function is only supported in WHERE commands")); + } + + public void testMultipleTerm() { + var query = """ + FROM test + | WHERE term(content,"fox") AND term(content,"brown") + | KEEP id + | SORT id + """; + + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValues(resp.values(), List.of(List.of(2), List.of(4), List.of(5))); + } + } + + public void testNotWhereTerm() { + var query = """ + FROM test + | WHERE NOT term(content,"brown") + | KEEP id + | SORT id + """; + + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValues(resp.values(), List.of(List.of(3))); + } + } + + private void createAndPopulateIndex() { + var indexName = "test"; + var client = client().admin().indices(); + var CreateRequest = client.prepareCreate(indexName) + .setSettings(Settings.builder().put("index.number_of_shards", 1)) + .setMapping("id", "type=integer", "content", "type=text"); + assertAcked(CreateRequest); + client().prepareBulk() + .add( + new IndexRequest(indexName).id("1") + .source("id", 1, "content", "The quick brown animal swiftly jumps over a lazy dog", "title", "A Swift Fox's Journey") + ) + .add( + new IndexRequest(indexName).id("2") + .source("id", 2, "content", "A speedy brown fox hops effortlessly over a sluggish canine", "title", "The Fox's Leap") + ) + .add( + new IndexRequest(indexName).id("3") + .source("id", 3, "content", "Quick and nimble, the fox vaults over the lazy dog", "title", "Brown Fox in Action") + ) + .add( + new IndexRequest(indexName).id("4") + .source( + "id", + 4, + "content", + "A fox that is quick and brown jumps over a dog that is quite lazy", + "title", + "Speedy Animals" + ) + ) + .add( + new IndexRequest(indexName).id("5") + .source( + "id", + 5, + "content", + "With agility, a quick brown fox bounds over a slow-moving dog", + "title", + "Foxes and Canines" + ) + ) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + ensureYellow(indexName); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 19ba6a5151eaf..ee3f5be185b4f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -550,7 +550,12 @@ public enum Cap { /** * Support the "METADATA _score" directive to enable _score column. */ - METADATA_SCORE(Build.current().isSnapshot()); + METADATA_SCORE(Build.current().isSnapshot()), + + /** + * Term function + */ + TERM_FUNCTION(Build.current().isSnapshot()); private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index f5fd82d742bc7..d6f0ff766eb40 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Term; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg; @@ -837,6 +838,14 @@ private static void checkFullTextQueryFunctions(LogicalPlan plan, Set f m -> "[" + m.functionName() + "] " + m.functionType(), failures ); + checkCommandsBeforeExpression( + plan, + condition, + Term.class, + lp -> (lp instanceof Limit == false) && (lp instanceof Aggregate == false), + m -> "[" + m.functionName() + "] " + m.functionType(), + failures + ); checkNotPresentInDisjunctions(condition, ftf -> "[" + ftf.functionName() + "] " + ftf.functionType(), failures); checkFullTextFunctionsParents(condition, failures); } else { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index c66a5293eb14a..3749b46879354 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Term; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; @@ -424,7 +425,8 @@ private static FunctionDefinition[][] snapshotFunctions() { // This is an experimental function and can be removed without notice. def(Delay.class, Delay::new, "delay"), def(Kql.class, Kql::new, "kql"), - def(Rate.class, Rate::withUnresolvedTimestamp, "rate") } }; + def(Rate.class, Rate::withUnresolvedTimestamp, "rate"), + def(Term.class, Term::new, "term") } }; } public EsqlFunctionRegistry snapshotRegistry() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java index 8804a031de78c..d6b79d16b74f6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java @@ -29,6 +29,9 @@ public static List getNamedWriteables() { if (EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()) { entries.add(Kql.ENTRY); } + if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { + entries.add(Term.ENTRY); + } return Collections.unmodifiableList(entries); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java new file mode 100644 index 0000000000000..125a5b02b6e1c --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.common.Failure; +import org.elasticsearch.xpack.esql.common.Failures; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.querydsl.query.TermQuery; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNull; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +/** + * Full text function that performs a {@link TermQuery} . + */ +public class Term extends FullTextFunction implements Validatable { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Term", Term::readFrom); + + private final Expression field; + + @FunctionInfo( + returnType = "boolean", + preview = true, + description = "Performs a Term query on the specified field. Returns true if the provided term matches the row.", + examples = { @Example(file = "term-function", tag = "term-with-field") } + ) + public Term( + Source source, + @Param(name = "field", type = { "keyword", "text" }, description = "Field that the query will target.") Expression field, + @Param( + name = "query", + type = { "keyword", "text" }, + description = "Term you wish to find in the provided field." + ) Expression termQuery + ) { + super(source, termQuery, List.of(field, termQuery)); + this.field = field; + } + + private static Term readFrom(StreamInput in) throws IOException { + Source source = Source.readFrom((PlanStreamInput) in); + Expression field = in.readNamedWriteable(Expression.class); + Expression query = in.readNamedWriteable(Expression.class); + return new Term(source, field, query); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(field()); + out.writeNamedWriteable(query()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveNonQueryParamTypes() { + return isNotNull(field, sourceText(), FIRST).and(isString(field, sourceText(), FIRST)).and(super.resolveNonQueryParamTypes()); + } + + @Override + public void validate(Failures failures) { + if (field instanceof FieldAttribute == false) { + failures.add( + Failure.fail( + field, + "[{}] {} cannot operate on [{}], which is not a field from an index mapping", + functionName(), + functionType(), + field.sourceText() + ) + ); + } + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Term(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Term::new, field, query()); + } + + protected TypeResolutions.ParamOrdinal queryParamOrdinal() { + return SECOND; + } + + public Expression field() { + return field; + } + + @Override + public String functionName() { + return ENTRY.name; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index 3d6c35e914294..9d02af0efbab0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.esql.core.util.Queries; import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Term; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; @@ -254,6 +255,8 @@ static boolean canPushToSource(Expression exp, LucenePushdownPredicates lucenePu return canPushSpatialFunctionToSource(spatial, lucenePushdownPredicates); } else if (exp instanceof Match mf) { return mf.field() instanceof FieldAttribute && DataType.isString(mf.field().dataType()); + } else if (exp instanceof Term term) { + return term.field() instanceof FieldAttribute && DataType.isString(term.field().dataType()); } else if (exp instanceof FullTextFunction) { return true; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index 1580b77931240..1aee8f029e474 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Term; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils; @@ -92,6 +93,7 @@ public final class EsqlExpressionTranslators { new MatchFunctionTranslator(), new QueryStringFunctionTranslator(), new KqlFunctionTranslator(), + new TermFunctionTranslator(), new Scalars() ); @@ -548,4 +550,12 @@ protected Query asQuery(Kql kqlFunction, TranslatorHandler handler) { return new KqlQuery(kqlFunction.source(), kqlFunction.queryAsText()); } } + + public static class TermFunctionTranslator extends ExpressionTranslator { + @Override + protected Query asQuery(Term term, TranslatorHandler handler) { + return new TermQuery(term.source(), ((FieldAttribute) term.field()).name(), term.queryAsText()); + } + } + } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 2e8b856cf82a6..2834e5f3f8358 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -265,6 +265,10 @@ public final void test() throws Throwable { "lookup join disabled for csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.JOIN_LOOKUP_V4.capabilityName()) ); + assumeFalse( + "can't use TERM function in csv tests", + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.TERM_FUNCTION.capabilityName()) + ); if (Build.current().isSnapshot()) { assertThat( "Capability is not included in the enabled list capabilities on a snapshot build. Spelling mistake?", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 882b8b7dbfd7c..7e3ef4f1f5f87 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1337,6 +1337,11 @@ public void testMatchFunctionOnlyAllowedInWhere() throws Exception { checkFullTextFunctionsOnlyAllowedInWhere("MATCH", "match(first_name, \"Anna\")", "function"); } + public void testTermFunctionOnlyAllowedInWhere() throws Exception { + assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); + checkFullTextFunctionsOnlyAllowedInWhere("Term", "term(first_name, \"Anna\")", "function"); + } + public void testMatchOperatornOnlyAllowedInWhere() throws Exception { checkFullTextFunctionsOnlyAllowedInWhere(":", "first_name:\"Anna\"", "operator"); } @@ -1401,6 +1406,11 @@ public void testMatchFunctionWithDisjunctions() { checkWithDisjunctions("MATCH", "match(first_name, \"Anna\")", "function"); } + public void testTermFunctionWithDisjunctions() { + assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); + checkWithDisjunctions("Term", "term(first_name, \"Anna\")", "function"); + } + public void testMatchOperatorWithDisjunctions() { checkWithDisjunctions(":", "first_name : \"Anna\"", "operator"); } @@ -1463,6 +1473,11 @@ public void testMatchFunctionWithNonBooleanFunctions() { checkFullTextFunctionsWithNonBooleanFunctions("MATCH", "match(first_name, \"Anna\")", "function"); } + public void testTermFunctionWithNonBooleanFunctions() { + assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); + checkFullTextFunctionsWithNonBooleanFunctions("Term", "term(first_name, \"Anna\")", "function"); + } + public void testMatchOperatorWithNonBooleanFunctions() { checkFullTextFunctionsWithNonBooleanFunctions(":", "first_name:\"Anna\"", "operator"); } @@ -1563,6 +1578,45 @@ public void testMatchTargetsExistingField() throws Exception { assertEquals("1:33: Unknown column [first_name]", error("from test | keep emp_no | where first_name : \"Anna\"")); } + public void testTermFunctionArgNotConstant() throws Exception { + assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); + assertEquals( + "1:19: second argument of [term(first_name, first_name)] must be a constant, received [first_name]", + error("from test | where term(first_name, first_name)") + ); + assertEquals( + "1:59: second argument of [term(first_name, query)] must be a constant, received [query]", + error("from test | eval query = concat(\"first\", \" name\") | where term(first_name, query)") + ); + // Other value types are tested in QueryStringFunctionTests + } + + // These should pass eventually once we lift some restrictions on match function + public void testTermFunctionCurrentlyUnsupportedBehaviour() throws Exception { + assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); + assertEquals( + "1:67: Unknown column [first_name]", + error("from test | stats max_salary = max(salary) by emp_no | where term(first_name, \"Anna\")") + ); + } + + public void testTermFunctionNullArgs() throws Exception { + assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); + assertEquals( + "1:19: first argument of [term(null, \"query\")] cannot be null, received [null]", + error("from test | where term(null, \"query\")") + ); + assertEquals( + "1:19: second argument of [term(first_name, null)] cannot be null, received [null]", + error("from test | where term(first_name, null)") + ); + } + + public void testTermTargetsExistingField() throws Exception { + assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); + assertEquals("1:38: Unknown column [first_name]", error("from test | keep emp_no | where term(first_name, \"Anna\")")); + } + public void testCoalesceWithMixedNumericTypes() { assertEquals( "1:22: second argument of [coalesce(languages, height)] must be [integer], found value [height] type [double]", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/TermTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/TermTests.java new file mode 100644 index 0000000000000..c1c0dc26880ab --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/TermTests.java @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +@FunctionName("term") +public class TermTests extends AbstractFunctionTestCase { + + public TermTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List> supportedPerPosition = supportedParams(); + List suppliers = new LinkedList<>(); + for (DataType fieldType : DataType.stringTypes()) { + for (DataType queryType : DataType.stringTypes()) { + addPositiveTestCase(List.of(fieldType, queryType), suppliers); + addNonFieldTestCase(List.of(fieldType, queryType), supportedPerPosition, suppliers); + } + } + + List suppliersWithErrors = errorsForCasesWithoutExamples(suppliers, (v, p) -> "string"); + + // Don't test null, as it is not allowed but the expected message is not a type error - so we check it separately in VerifierTests + return parameterSuppliersFromTypedData( + suppliersWithErrors.stream().filter(s -> s.types().contains(DataType.NULL) == false).toList() + ); + } + + protected static List> supportedParams() { + Set supportedTextParams = Set.of(DataType.KEYWORD, DataType.TEXT); + Set supportedNumericParams = Set.of(DataType.DOUBLE, DataType.INTEGER); + Set supportedFuzzinessParams = Set.of(DataType.INTEGER, DataType.KEYWORD, DataType.TEXT); + List> supportedPerPosition = List.of( + supportedTextParams, + supportedTextParams, + supportedNumericParams, + supportedFuzzinessParams + ); + return supportedPerPosition; + } + + protected static void addPositiveTestCase(List paramDataTypes, List suppliers) { + + // Positive case - creates an ES field from the field parameter type + suppliers.add( + new TestCaseSupplier( + getTestCaseName(paramDataTypes, "-ES field"), + paramDataTypes, + () -> new TestCaseSupplier.TestCase( + getTestParams(paramDataTypes), + "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]", + DataType.BOOLEAN, + equalTo(true) + ) + ) + ); + } + + private static void addNonFieldTestCase( + List paramDataTypes, + List> supportedPerPosition, + List suppliers + ) { + // Negative case - use directly the field parameter type + suppliers.add( + new TestCaseSupplier( + getTestCaseName(paramDataTypes, "-non ES field"), + paramDataTypes, + typeErrorSupplier(true, supportedPerPosition, paramDataTypes, TermTests::matchTypeErrorSupplier) + ) + ); + } + + private static List getTestParams(List paramDataTypes) { + String fieldName = randomIdentifier(); + List params = new ArrayList<>(); + params.add( + new TestCaseSupplier.TypedData( + new FieldExpression(fieldName, List.of(new FieldExpression.FieldValue(fieldName))), + paramDataTypes.get(0), + "field" + ) + ); + params.add(new TestCaseSupplier.TypedData(new BytesRef(randomAlphaOfLength(10)), paramDataTypes.get(1), "query")); + return params; + } + + private static String getTestCaseName(List paramDataTypes, String fieldType) { + StringBuilder sb = new StringBuilder(); + sb.append("<"); + sb.append(paramDataTypes.get(0)).append(fieldType).append(", "); + sb.append(paramDataTypes.get(1)); + sb.append(">"); + return sb.toString(); + } + + private static String matchTypeErrorSupplier(boolean includeOrdinal, List> validPerPosition, List types) { + return "[] cannot operate on [" + types.getFirst().typeName() + "], which is not a field from an index mapping"; + } + + @Override + protected Expression build(Source source, List args) { + return new Match(source, args.get(0), args.get(1)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 86f5c812737b1..d32124c1aaf32 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -1391,6 +1391,35 @@ public void testMultipleMatchFilterPushdown() { assertThat(actualLuceneQuery.toString(), is(expectedLuceneQuery.toString())); } + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"term":{"last_name":{"query":"Smith"}}}] + */ + public void testTermFunction() { + // Skip test if the term function is not enabled. + assumeTrue("term function capability not available", EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()); + + var plan = plannerOptimizer.plan(""" + from test + | where term(last_name, "Smith") + """, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + var expected = QueryBuilders.termQuery("last_name", "Smith"); + assertThat(query.query().toString(), is(expected.toString())); + } + private QueryBuilder wrapWithSingleQuery(String query, QueryBuilder inner, String fieldName, Source source) { return FilterTests.singleValueQuery(query, inner, fieldName, source); } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index e6c061f44a9e4..81f65668722fc 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -92,7 +92,7 @@ setup: - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} # Testing for the entire function set isn't feasbile, so we just check that we return the correct count as an approximation. - - length: {esql.functions: 127} # check the "sister" test below for a likely update to the same esql.functions length check + - length: {esql.functions: 128} # check the "sister" test below for a likely update to the same esql.functions length check --- "Basic ESQL usage output (telemetry) non-snapshot version": From f27cb5efd3ffaf9bfe9a2f553aa9abdc39c9b15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Fri, 6 Dec 2024 09:15:15 +0100 Subject: [PATCH 067/204] [DOCS] Adds examples to inference processor docs (#116018) --- .../ingest/processors/inference.asciidoc | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc index 9c6f0592a1d91..e079b9d665290 100644 --- a/docs/reference/ingest/processors/inference.asciidoc +++ b/docs/reference/ingest/processors/inference.asciidoc @@ -735,3 +735,70 @@ You can also specify the target field as follows: In this case, {feat-imp} is exposed in the `my_field.foo.feature_importance` field. + + +[discrete] +[[inference-processor-examples]] +==== {infer-cap} processor examples + +The following example uses an <> in an {infer} processor named `query_helper_pipeline` to perform a chat completion task. +The processor generates an {es} query from natural language input using a prompt designed for a completion task type. +Refer to <> for the {infer} service you use and check the corresponding examples of setting up an endpoint with the chat completion task type. + + +[source,console] +-------------------------------------------------- +PUT _ingest/pipeline/query_helper_pipeline +{ + "processors": [ + { + "script": { + "source": "ctx.prompt = 'Please generate an elasticsearch search query on index `articles_index` for the following natural language query. Dates are in the field `@timestamp`, document types are in the field `type` (options are `news`, `publication`), categories in the field `category` and can be multiple (options are `medicine`, `pharmaceuticals`, `technology`), and document names are in the field `title` which should use a fuzzy match. Ignore fields which cannot be determined from the natural language query context: ' + ctx.content" <1> + } + }, + { + "inference": { + "model_id": "openai_chat_completions", <2> + "input_output": { + "input_field": "prompt", + "output_field": "query" + } + } + }, + { + "remove": { + "field": "prompt" + } + } + ] +} +-------------------------------------------------- +// TEST[skip: An inference endpoint is required.] +<1> The `prompt` field contains the prompt used for the completion task, created with <>. +`+ ctx.content` appends the natural language input to the prompt. +<2> The ID of the pre-configured {infer} endpoint, which utilizes the <> with the `completion` task type. + +The following API request will simulate running a document through the ingest pipeline created previously: + +[source,console] +-------------------------------------------------- +POST _ingest/pipeline/query_helper_pipeline/_simulate +{ + "docs": [ + { + "_source": { + "content": "artificial intelligence in medicine articles published in the last 12 months" <1> + } + } + ] +} +-------------------------------------------------- +// TEST[skip: An inference processor with an inference endpoint is required.] +<1> The natural language query used to generate an {es} query within the prompt created by the {infer} processor. + + +[discrete] +[[infer-proc-readings]] +==== Further readings + +* https://www.elastic.co/search-labs/blog/openwebcrawler-llms-semantic-text-resume-job-search[Which job is the best for you? Using LLMs and semantic_text to match resumes to jobs] \ No newline at end of file From 0a2c9fbc2925adac6057b86c4ea9d6174121c6ef Mon Sep 17 00:00:00 2001 From: Matteo Piergiovanni <134913285+piergm@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:59:19 +0100 Subject: [PATCH 068/204] fixes and unmutes testSearchableSnapshotShardsAreSkipped... (#118133) --- muted-tests.yml | 3 --- .../SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 99d1e170a4f4e..e5199b0a1e238 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -132,9 +132,6 @@ tests: - class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests method: testInvalidJSON issue: https://github.com/elastic/elasticsearch/issues/116521 -- class: org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsCanMatchOnCoordinatorIntegTests - method: testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQueryingAnyNodeWhenTheyAreOutsideOfTheQueryRange - issue: https://github.com/elastic/elasticsearch/issues/116523 - class: org.elasticsearch.reservedstate.service.RepositoriesFileSettingsIT method: testSettingsApplied issue: https://github.com/elastic/elasticsearch/issues/116694 diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java index 21b24db6ce8d5..d4bbd4495df26 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java @@ -379,7 +379,7 @@ public void testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQuerying } if (searchShardsResponse != null) { for (SearchShardsGroup group : searchShardsResponse.getGroups()) { - assertFalse("no shard should be marked as skipped", group.skipped()); + assertTrue("the shard is skipped because index value is outside the query time range", group.skipped()); } } } From e9d925e12728943b14f3d73e9db2ec41d1d161e5 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Fri, 6 Dec 2024 11:15:49 +0100 Subject: [PATCH 069/204] ES|QL: make ignoreOrder parsing more strict in CSV tests (#118136) --- muted-tests.yml | 2 -- .../java/org/elasticsearch/xpack/esql/CsvSpecReader.java | 7 ++++++- .../testFixtures/src/main/resources/lookup-join.csv-spec | 8 ++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index e5199b0a1e238..e4cd94b221536 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -242,8 +242,6 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveTests method: test40AutoconfigurationNotTriggeredWhenNodeIsMeantToJoinExistingCluster issue: https://github.com/elastic/elasticsearch/issues/118029 -- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT - issue: https://github.com/elastic/elasticsearch/issues/117981 - class: org.elasticsearch.packaging.test.ConfigurationTests method: test30SymlinkedDataPath issue: https://github.com/elastic/elasticsearch/issues/118111 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvSpecReader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvSpecReader.java index 84e06e0c1b674..ba0d11059a69b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvSpecReader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvSpecReader.java @@ -80,7 +80,12 @@ public Object parse(String line) { testCase.expectedWarningsRegexString.add(regex); testCase.expectedWarningsRegex.add(warningRegexToPattern(regex)); } else if (lower.startsWith("ignoreorder:")) { - testCase.ignoreOrder = Boolean.parseBoolean(line.substring("ignoreOrder:".length()).trim()); + String value = lower.substring("ignoreOrder:".length()).trim(); + if ("true".equals(value)) { + testCase.ignoreOrder = true; + } else if ("false".equals(value) == false) { + throw new IllegalArgumentException("Invalid value for ignoreOrder: [" + value + "], it can only be true or false"); + } } else if (line.startsWith(";")) { testCase.expectedResults = data.toString(); // clean-up and emit diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 584cde55080ef..2d4c105cfff20 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -140,7 +140,7 @@ FROM sample_data | EVAL client_ip = client_ip::keyword | LOOKUP JOIN clientips_lookup ON client_ip ; -ignoreOrder:true; +ignoreOrder:true @timestamp:date | event_duration:long | message:keyword | client_ip:keyword | env:keyword 2023-10-23T13:55:01.543Z | 1756467 | Connected to 10.1.0.1 | 172.21.3.15 | Production @@ -160,7 +160,7 @@ FROM sample_data | LOOKUP JOIN clientips_lookup ON client_ip | KEEP @timestamp, client_ip, event_duration, message, env ; -ignoreOrder:true; +ignoreOrder:true @timestamp:date | client_ip:keyword | event_duration:long | message:keyword | env:keyword 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Production @@ -245,7 +245,7 @@ required_capability: join_lookup_v4 FROM sample_data | LOOKUP JOIN message_types_lookup ON message ; -ignoreOrder:true; +ignoreOrder:true @timestamp:date | client_ip:ip | event_duration:long | message:keyword | type:keyword 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Success @@ -264,7 +264,7 @@ FROM sample_data | LOOKUP JOIN message_types_lookup ON message | KEEP @timestamp, client_ip, event_duration, message, type ; -ignoreOrder:true; +ignoreOrder:true @timestamp:date | client_ip:ip | event_duration:long | message:keyword | type:keyword 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Success From 34ea8f365f16b70a31693e6ce8dda3a8dfd13d92 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 6 Dec 2024 12:40:58 +0200 Subject: [PATCH 070/204] Search Queries in parallel assertHitcount - part 1 (#117467) Update tests applying an optimization in assertions --- .../elasticsearch/cluster/NoMasterNodeIT.java | 9 +-- .../indices/IndicesOptionsIntegrationIT.java | 7 ++- .../recovery/IndexPrimaryRelocationIT.java | 6 +- .../template/SimpleIndexTemplateIT.java | 7 +-- .../search/nested/SimpleNestedIT.java | 24 ++++---- .../search/scroll/SearchScrollIT.java | 32 +++++++--- .../search/simple/SimpleSearchIT.java | 61 +++++++++++-------- .../snapshots/RestoreSnapshotIT.java | 21 +++++-- 8 files changed, 99 insertions(+), 68 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/NoMasterNodeIT.java index 13515d34ec65f..545b38f30ba94 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/NoMasterNodeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/NoMasterNodeIT.java @@ -261,10 +261,11 @@ public void testNoMasterActionsWriteMasterBlock() throws Exception { GetResponse getResponse = clientToMasterlessNode.prepareGet("test1", "1").get(); assertExists(getResponse); - assertHitCount(clientToMasterlessNode.prepareSearch("test1").setAllowPartialSearchResults(true).setSize(0), 1L); - - logger.info("--> here 3"); - assertHitCount(clientToMasterlessNode.prepareSearch("test1").setAllowPartialSearchResults(true), 1L); + assertHitCount( + 1L, + clientToMasterlessNode.prepareSearch("test1").setAllowPartialSearchResults(true).setSize(0), + clientToMasterlessNode.prepareSearch("test1").setAllowPartialSearchResults(true) + ); assertResponse(clientToMasterlessNode.prepareSearch("test2").setAllowPartialSearchResults(true).setSize(0), countResponse -> { assertThat(countResponse.getTotalShards(), equalTo(3)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java index f41277c5b80ca..545ed83bb79c8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java @@ -398,8 +398,11 @@ public void testWildcardBehaviourSnapshotRestore() throws Exception { public void testAllMissingLenient() throws Exception { createIndex("test1"); prepareIndex("test1").setId("1").setSource("k", "v").setRefreshPolicy(IMMEDIATE).get(); - assertHitCount(prepareSearch("test2").setIndicesOptions(IndicesOptions.lenientExpandOpen()).setQuery(matchAllQuery()), 0L); - assertHitCount(prepareSearch("test2", "test3").setQuery(matchAllQuery()).setIndicesOptions(IndicesOptions.lenientExpandOpen()), 0L); + assertHitCount( + 0L, + prepareSearch("test2").setIndicesOptions(IndicesOptions.lenientExpandOpen()).setQuery(matchAllQuery()), + prepareSearch("test2", "test3").setQuery(matchAllQuery()).setIndicesOptions(IndicesOptions.lenientExpandOpen()) + ); // you should still be able to run empty searches without things blowing up assertHitCount(prepareSearch().setIndicesOptions(IndicesOptions.lenientExpandOpen()).setQuery(matchAllQuery()), 1L); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java index 581145d949cf9..debcf5c06a7d6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java @@ -98,11 +98,11 @@ public void run() { finished.set(true); indexingThread.join(); refresh("test"); - ElasticsearchAssertions.assertHitCount(prepareSearch("test").setTrackTotalHits(true), numAutoGenDocs.get()); ElasticsearchAssertions.assertHitCount( + numAutoGenDocs.get(), + prepareSearch("test").setTrackTotalHits(true), prepareSearch("test").setTrackTotalHits(true)// extra paranoia ;) - .setQuery(QueryBuilders.termQuery("auto", true)), - numAutoGenDocs.get() + .setQuery(QueryBuilders.termQuery("auto", true)) ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index de9e3f28a2109..8496180e85d4e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -500,9 +500,7 @@ public void testIndexTemplateWithAliases() throws Exception { refresh(); - assertHitCount(prepareSearch("test_index"), 5L); - assertHitCount(prepareSearch("simple_alias"), 5L); - assertHitCount(prepareSearch("templated_alias-test_index"), 5L); + assertHitCount(5L, prepareSearch("test_index"), prepareSearch("simple_alias"), prepareSearch("templated_alias-test_index")); assertResponse(prepareSearch("filtered_alias"), response -> { assertHitCount(response, 1L); @@ -584,8 +582,7 @@ public void testIndexTemplateWithAliasesSource() { prepareIndex("test_index").setId("2").setSource("field", "value2").get(); refresh(); - assertHitCount(prepareSearch("test_index"), 2L); - assertHitCount(prepareSearch("alias1"), 2L); + assertHitCount(2L, prepareSearch("test_index"), prepareSearch("alias1")); assertResponse(prepareSearch("alias2"), response -> { assertHitCount(response, 1L); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java index 8225386ed02d2..acfc55a740f1e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java @@ -53,6 +53,7 @@ import static org.hamcrest.Matchers.startsWith; public class SimpleNestedIT extends ESIntegTestCase { + public void testSimpleNested() throws Exception { assertAcked(prepareCreate("test").setMapping("nested1", "type=nested")); ensureGreen(); @@ -87,21 +88,20 @@ public void testSimpleNested() throws Exception { // check the numDocs assertDocumentCount("test", 3); - assertHitCount(prepareSearch("test").setQuery(termQuery("n_field1", "n_value1_1")), 0L); - - // search for something that matches the nested doc, and see that we don't find the nested doc - assertHitCount(prepareSearch("test"), 1L); - assertHitCount(prepareSearch("test").setQuery(termQuery("n_field1", "n_value1_1")), 0L); + assertHitCount( + 0L, + prepareSearch("test").setQuery(termQuery("n_field1", "n_value1_1")), + prepareSearch("test").setQuery(termQuery("n_field1", "n_value1_1")) + ); - // now, do a nested query - assertHitCountAndNoFailures( + assertHitCount( + 1L, + // search for something that matches the nested doc, and see that we don't find the nested doc + prepareSearch("test"), + // now, do a nested query prepareSearch("test").setQuery(nestedQuery("nested1", termQuery("nested1.n_field1", "n_value1_1"), ScoreMode.Avg)), - 1L - ); - assertHitCountAndNoFailures( prepareSearch("test").setQuery(nestedQuery("nested1", termQuery("nested1.n_field1", "n_value1_1"), ScoreMode.Avg)) - .setSearchType(SearchType.DFS_QUERY_THEN_FETCH), - 1L + .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) ); // add another doc, one that would match if it was not nested... diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java index 7ac24b77a4b6d..a54e19b839ad3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java @@ -206,11 +206,17 @@ public void testScrollAndUpdateIndex() throws Exception { indicesAdmin().prepareRefresh().get(); - assertHitCount(prepareSearch().setSize(0).setQuery(matchAllQuery()), 500); - assertHitCount(prepareSearch().setSize(0).setQuery(termQuery("message", "test")), 500); - assertHitCount(prepareSearch().setSize(0).setQuery(termQuery("message", "test")), 500); - assertHitCount(prepareSearch().setSize(0).setQuery(termQuery("message", "update")), 0); - assertHitCount(prepareSearch().setSize(0).setQuery(termQuery("message", "update")), 0); + assertHitCount( + 500, + prepareSearch().setSize(0).setQuery(matchAllQuery()), + prepareSearch().setSize(0).setQuery(termQuery("message", "test")), + prepareSearch().setSize(0).setQuery(termQuery("message", "test")) + ); + assertHitCount( + 0, + prepareSearch().setSize(0).setQuery(termQuery("message", "update")), + prepareSearch().setSize(0).setQuery(termQuery("message", "update")) + ); SearchResponse searchResponse = prepareSearch().setQuery(queryStringQuery("user:kimchy")) .setSize(35) @@ -229,11 +235,17 @@ public void testScrollAndUpdateIndex() throws Exception { } while (searchResponse.getHits().getHits().length > 0); indicesAdmin().prepareRefresh().get(); - assertHitCount(prepareSearch().setSize(0).setQuery(matchAllQuery()), 500); - assertHitCount(prepareSearch().setSize(0).setQuery(termQuery("message", "test")), 0); - assertHitCount(prepareSearch().setSize(0).setQuery(termQuery("message", "test")), 0); - assertHitCount(prepareSearch().setSize(0).setQuery(termQuery("message", "update")), 500); - assertHitCount(prepareSearch().setSize(0).setQuery(termQuery("message", "update")), 500); + assertHitCount( + 500, + prepareSearch().setSize(0).setQuery(matchAllQuery()), + prepareSearch().setSize(0).setQuery(termQuery("message", "update")), + prepareSearch().setSize(0).setQuery(termQuery("message", "update")) + ); + assertHitCount( + 0, + prepareSearch().setSize(0).setQuery(termQuery("message", "test")), + prepareSearch().setSize(0).setQuery(termQuery("message", "test")) + ); } finally { clearScroll(searchResponse.getScrollId()); searchResponse.decRef(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java index e87c4790aa665..5a9be73d92268 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -147,16 +147,22 @@ public void testIpCidr() throws Exception { prepareIndex("test").setId("5").setSource("ip", "2001:db8::ff00:42:8329").get(); refresh(); - assertHitCount(prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1"))), 1L); - assertHitCount(prepareSearch().setQuery(queryStringQuery("ip: 192.168.0.1")), 1L); - assertHitCount(prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1/32"))), 1L); + assertHitCount(prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.1.5/32"))), 0L); + assertHitCount( + 1L, + prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1"))), + prepareSearch().setQuery(queryStringQuery("ip: 192.168.0.1")), + prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1/32"))), + prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "2001:db8::ff00:42:8329/128"))), + prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "2001:db8::/64"))) + ); assertHitCount(prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.0/24"))), 3L); - assertHitCount(prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.0.0.0/8"))), 4L); - assertHitCount(prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0.0.0.0/0"))), 4L); - assertHitCount(prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "2001:db8::ff00:42:8329/128"))), 1L); - assertHitCount(prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "2001:db8::/64"))), 1L); + assertHitCount( + 4L, + prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.0.0.0/8"))), + prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0.0.0.0/0"))) + ); assertHitCount(prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "::/0"))), 5L); - assertHitCount(prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.1.5/32"))), 0L); assertFailures( prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0/0/0/0/0"))), @@ -170,8 +176,11 @@ public void testSimpleId() { prepareIndex("test").setId("XXX1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); // id is not indexed, but lets see that we automatically convert to - assertHitCount(prepareSearch().setQuery(QueryBuilders.termQuery("_id", "XXX1")), 1L); - assertHitCount(prepareSearch().setQuery(QueryBuilders.queryStringQuery("_id:XXX1")), 1L); + assertHitCount( + 1L, + prepareSearch().setQuery(QueryBuilders.termQuery("_id", "XXX1")), + prepareSearch().setQuery(QueryBuilders.queryStringQuery("_id:XXX1")) + ); } public void testSimpleDateRange() throws Exception { @@ -324,12 +333,12 @@ public void testLargeFromAndSizeSucceeds() throws Exception { createIndex("idx"); indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); - assertHitCount(prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - 10), 1); - assertHitCount(prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)), 1); assertHitCount( + 1, + prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - 10), + prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)), prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2 - 1), - 1 + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2 - 1) ); } @@ -340,12 +349,12 @@ public void testTooLargeFromAndSizeOkBySetting() throws Exception { ).get(); indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); - assertHitCount(prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)), 1); - assertHitCount(prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1), 1); assertHitCount( + 1, + prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)), + prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1), prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)), - 1 + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) ); } @@ -358,12 +367,12 @@ public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception { ); indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); - assertHitCount(prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)), 1); - assertHitCount(prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1), 1); assertHitCount( + 1, + prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)), + prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1), prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)), - 1 + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)) ); } @@ -371,12 +380,12 @@ public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws prepareCreate("idx").setSettings(Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE)).get(); indexRandom(true, prepareIndex("idx").setSource("{}", XContentType.JSON)); - assertHitCount(prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10), 1); - assertHitCount(prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10), 1); assertHitCount( + 1, + prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10), + prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10), prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10) - .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10), - 1 + .setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10) ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java index fe83073eeb780..b490c7efd52cd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java @@ -678,9 +678,12 @@ public void testChangeSettingsOnRestore() throws Exception { indexRandom(true, builders); flushAndRefresh(); - assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "foo")), numdocs); + assertHitCount( + numdocs, + client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "foo")), + client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "bar")) + ); assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "Foo")), 0); - assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "bar")), numdocs); createSnapshot("test-repo", "test-snap", Collections.singletonList("test-idx")); @@ -736,8 +739,11 @@ public void testChangeSettingsOnRestore() throws Exception { assertThat(getSettingsResponse.getSetting("test-idx", SETTING_NUMBER_OF_SHARDS), equalTo("" + numberOfShards)); assertThat(getSettingsResponse.getSetting("test-idx", "index.analysis.analyzer.my_analyzer.type"), equalTo("standard")); - assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "Foo")), numdocs); - assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "bar")), numdocs); + assertHitCount( + numdocs, + client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "Foo")), + client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "bar")) + ); logger.info("--> delete the index and recreate it while deleting all index settings"); cluster().wipeIndices("test-idx"); @@ -758,8 +764,11 @@ public void testChangeSettingsOnRestore() throws Exception { // Make sure that number of shards didn't change assertThat(getSettingsResponse.getSetting("test-idx", SETTING_NUMBER_OF_SHARDS), equalTo("" + numberOfShards)); - assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "Foo")), numdocs); - assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "bar")), numdocs); + assertHitCount( + numdocs, + client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "Foo")), + client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "bar")) + ); } public void testRestoreChangeIndexMode() { From a0ac839189431e016ea810ac1e48b6932b6daa0b Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 6 Dec 2024 11:02:17 +0000 Subject: [PATCH 071/204] [ML] Wait for the worker service to shutdown before closing task processor (#117920) --- docs/changelog/117920.yaml | 6 ++++ .../deployment/DeploymentManager.java | 33 +++++++------------ 2 files changed, 18 insertions(+), 21 deletions(-) create mode 100644 docs/changelog/117920.yaml diff --git a/docs/changelog/117920.yaml b/docs/changelog/117920.yaml new file mode 100644 index 0000000000000..1bfddabd4462d --- /dev/null +++ b/docs/changelog/117920.yaml @@ -0,0 +1,6 @@ +pr: 117920 +summary: Wait for the worker service to shutdown before closing task processor +area: Machine Learning +type: bug +issues: + - 117563 diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java index 9187969fc25a4..c6f1ebcc10780 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java @@ -631,12 +631,15 @@ synchronized void forcefullyStopProcess() { logger.debug(() -> format("[%s] Forcefully stopping process", task.getDeploymentId())); prepareInternalStateForShutdown(); - if (priorityProcessWorker.isShutdown()) { - // most likely there was a crash or exception that caused the - // thread to stop. Notify any waiting requests in the work queue - handleAlreadyShuttingDownWorker(); - } else { - priorityProcessWorker.shutdown(); + priorityProcessWorker.shutdownNow(); + try { + // wait for any currently executing work to finish + if (priorityProcessWorker.awaitTermination(10L, TimeUnit.SECONDS)) { + priorityProcessWorker.notifyQueueRunnables(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.info(Strings.format("[%s] Interrupted waiting for process worker after shutdownNow", PROCESS_NAME)); } killProcessIfPresent(); @@ -649,12 +652,6 @@ private void prepareInternalStateForShutdown() { stateStreamer.cancel(); } - private void handleAlreadyShuttingDownWorker() { - logger.debug(() -> format("[%s] Process worker was already marked for shutdown", task.getDeploymentId())); - - priorityProcessWorker.notifyQueueRunnables(); - } - private void killProcessIfPresent() { try { if (process.get() == null) { @@ -675,15 +672,7 @@ private void closeNlpTaskProcessor() { private synchronized void stopProcessAfterCompletingPendingWork() { logger.debug(() -> format("[%s] Stopping process after completing its pending work", task.getDeploymentId())); prepareInternalStateForShutdown(); - - if (priorityProcessWorker.isShutdown()) { - // most likely there was a crash or exception that caused the - // thread to stop. Notify any waiting requests in the work queue - handleAlreadyShuttingDownWorker(); - } else { - signalAndWaitForWorkerTermination(); - } - + signalAndWaitForWorkerTermination(); stopProcessGracefully(); closeNlpTaskProcessor(); } @@ -707,6 +696,8 @@ private void awaitTerminationAfterCompletingWork() throws TimeoutException { throw new TimeoutException( Strings.format("Timed out waiting for process worker to complete for process %s", PROCESS_NAME) ); + } else { + priorityProcessWorker.notifyQueueRunnables(); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); From 5b5c9adc47acd7f5dc385763fb6f5dafd586e18c Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Fri, 6 Dec 2024 07:22:21 -0500 Subject: [PATCH 072/204] Remove long deprecated special 'base' case for similarity service (#118115) --- .../index/similarity/SimilarityService.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java index 0f1b40f80c36c..9db316d9683ed 100644 --- a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java +++ b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java @@ -20,8 +20,6 @@ import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.TriFunction; -import org.elasticsearch.common.logging.DeprecationCategory; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; @@ -40,7 +38,6 @@ import java.util.function.Supplier; public final class SimilarityService { - private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(SimilarityService.class); public static final String DEFAULT_SIMILARITY = "BM25"; private static final Map>> DEFAULTS; public static final Map> BUILT_IN; @@ -115,13 +112,6 @@ public SimilarityService( defaultSimilarity = (providers.get("default") != null) ? providers.get("default").get() : providers.get(SimilarityService.DEFAULT_SIMILARITY).get(); - if (providers.get("base") != null) { - deprecationLogger.warn( - DeprecationCategory.QUERIES, - "base_similarity_ignored", - "The [base] similarity is ignored since query normalization and coords have been removed" - ); - } } /** From b09f1d72f40c78667f8128c5ef9e451fbe2b7b0e Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 6 Dec 2024 13:46:07 +0100 Subject: [PATCH 073/204] Add additional debug logging for indexing failure. (#117728) This change adds a new IndexingOperationListener that logs debug logging for indexing failures. Today `IndexShard` logger needs to be set to TRACE in order to see indexing failures. This logger is very verbose and logs many other aspects of indexing at trace level. This new logger is targeted just for seeing indexing failures. Relates #117700 --- .../elasticsearch/index/shard/IndexShard.java | 3 +- .../shard/IndexingFailuresDebugListener.java | 54 +++++++ .../IndexingFailuresDebugListenerTests.java | 138 ++++++++++++++++++ 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 server/src/main/java/org/elasticsearch/index/shard/IndexingFailuresDebugListener.java create mode 100644 server/src/test/java/org/elasticsearch/index/shard/IndexingFailuresDebugListenerTests.java diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 993079a3106d7..f84ac22cd78e4 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -345,8 +345,9 @@ public IndexShard( this.mapperService = mapperService; this.indexCache = indexCache; this.internalIndexingStats = new InternalIndexingStats(); + var indexingFailuresDebugListener = new IndexingFailuresDebugListener(this); this.indexingOperationListeners = new IndexingOperationListener.CompositeListener( - CollectionUtils.appendToCopyNoNullElements(listeners, internalIndexingStats), + CollectionUtils.appendToCopyNoNullElements(listeners, internalIndexingStats, indexingFailuresDebugListener), logger ); this.bulkOperationListener = new ShardBulkStats(); diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexingFailuresDebugListener.java b/server/src/main/java/org/elasticsearch/index/shard/IndexingFailuresDebugListener.java new file mode 100644 index 0000000000000..13c0d917d492d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexingFailuresDebugListener.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.shard; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.index.engine.Engine; + +import static org.elasticsearch.core.Strings.format; + +public class IndexingFailuresDebugListener implements IndexingOperationListener { + + private static final Logger LOGGER = LogManager.getLogger(IndexingFailuresDebugListener.class); + + private final IndexShard indexShard; + + public IndexingFailuresDebugListener(IndexShard indexShard) { + this.indexShard = indexShard; + } + + @Override + public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) { + if (LOGGER.isDebugEnabled()) { + if (result.getResultType() == Engine.Result.Type.FAILURE) { + postIndex(shardId, index, result.getFailure()); + } + } + } + + @Override + public void postIndex(ShardId shardId, Engine.Index index, Exception ex) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + () -> format( + "index-fail [%s] seq# [%s] allocation-id [%s] primaryTerm [%s] operationPrimaryTerm [%s] origin [%s]", + index.id(), + index.seqNo(), + indexShard.routingEntry().allocationId(), + index.primaryTerm(), + indexShard.getOperationPrimaryTerm(), + index.origin() + ), + ex + ); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexingFailuresDebugListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexingFailuresDebugListenerTests.java new file mode 100644 index 0000000000000..43434a691bd90 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexingFailuresDebugListenerTests.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.shard; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.MockAppender; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.EngineTestCase; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.test.ESTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class IndexingFailuresDebugListenerTests extends ESTestCase { + + static MockAppender appender; + static Logger testLogger1 = LogManager.getLogger(IndexingFailuresDebugListener.class); + static Level origLogLevel = testLogger1.getLevel(); + + @BeforeClass + public static void init() throws IllegalAccessException { + appender = new MockAppender("mock_appender"); + appender.start(); + Loggers.addAppender(testLogger1, appender); + Loggers.setLevel(testLogger1, randomBoolean() ? Level.DEBUG : Level.TRACE); + } + + @AfterClass + public static void cleanup() { + Loggers.removeAppender(testLogger1, appender); + appender.stop(); + + Loggers.setLevel(testLogger1, origLogLevel); + } + + public void testPostIndexException() { + var shardId = ShardId.fromString("[index][123]"); + var mockShard = mock(IndexShard.class); + var shardRouting = TestShardRouting.newShardRouting(shardId, "node-id", true, ShardRoutingState.STARTED); + when(mockShard.routingEntry()).thenReturn(shardRouting); + when(mockShard.getOperationPrimaryTerm()).thenReturn(1L); + IndexingFailuresDebugListener indexingFailuresDebugListener = new IndexingFailuresDebugListener(mockShard); + + ParsedDocument doc = EngineTestCase.createParsedDoc("1", null); + Engine.Index index = new Engine.Index(Uid.encodeId("doc_id"), 1, doc); + indexingFailuresDebugListener.postIndex(shardId, index, new RuntimeException("test exception")); + String message = appender.getLastEventAndReset().getMessage().getFormattedMessage(); + assertThat( + message, + equalTo( + "index-fail [1] seq# [-2] allocation-id [" + + shardRouting.allocationId() + + "] primaryTerm [1] operationPrimaryTerm [1] origin [PRIMARY]" + ) + ); + } + + public void testPostIndexExceptionInfoLevel() { + var previousLevel = testLogger1.getLevel(); + try { + Loggers.setLevel(testLogger1, randomBoolean() ? Level.INFO : Level.WARN); + var shardId = ShardId.fromString("[index][123]"); + var mockShard = mock(IndexShard.class); + var shardRouting = TestShardRouting.newShardRouting(shardId, "node-id", true, ShardRoutingState.STARTED); + when(mockShard.routingEntry()).thenReturn(shardRouting); + when(mockShard.getOperationPrimaryTerm()).thenReturn(1L); + IndexingFailuresDebugListener indexingFailuresDebugListener = new IndexingFailuresDebugListener(mockShard); + + ParsedDocument doc = EngineTestCase.createParsedDoc("1", null); + Engine.Index index = new Engine.Index(Uid.encodeId("doc_id"), 1, doc); + indexingFailuresDebugListener.postIndex(shardId, index, new RuntimeException("test exception")); + assertThat(appender.getLastEventAndReset(), nullValue()); + } finally { + Loggers.setLevel(testLogger1, previousLevel); + } + } + + public void testPostIndexFailure() { + var shardId = ShardId.fromString("[index][123]"); + var mockShard = mock(IndexShard.class); + var shardRouting = TestShardRouting.newShardRouting(shardId, "node-id", true, ShardRoutingState.STARTED); + when(mockShard.routingEntry()).thenReturn(shardRouting); + when(mockShard.getOperationPrimaryTerm()).thenReturn(1L); + IndexingFailuresDebugListener indexingFailuresDebugListener = new IndexingFailuresDebugListener(mockShard); + + ParsedDocument doc = EngineTestCase.createParsedDoc("1", null); + Engine.Index index = new Engine.Index(Uid.encodeId("doc_id"), 1, doc); + Engine.IndexResult indexResult = mock(Engine.IndexResult.class); + when(indexResult.getResultType()).thenReturn(Engine.Result.Type.FAILURE); + when(indexResult.getFailure()).thenReturn(new RuntimeException("test exception")); + indexingFailuresDebugListener.postIndex(shardId, index, indexResult); + String message = appender.getLastEventAndReset().getMessage().getFormattedMessage(); + assertThat( + message, + equalTo( + "index-fail [1] seq# [-2] allocation-id [" + + shardRouting.allocationId() + + "] primaryTerm [1] operationPrimaryTerm [1] origin [PRIMARY]" + ) + ); + } + + public void testPostIndex() { + var shardId = ShardId.fromString("[index][123]"); + var mockShard = mock(IndexShard.class); + var shardRouting = TestShardRouting.newShardRouting(shardId, "node-id", true, ShardRoutingState.STARTED); + when(mockShard.routingEntry()).thenReturn(shardRouting); + when(mockShard.getOperationPrimaryTerm()).thenReturn(1L); + IndexingFailuresDebugListener indexingFailuresDebugListener = new IndexingFailuresDebugListener(mockShard); + + ParsedDocument doc = EngineTestCase.createParsedDoc("1", null); + Engine.Index index = new Engine.Index(Uid.encodeId("doc_id"), 1, doc); + Engine.IndexResult indexResult = mock(Engine.IndexResult.class); + when(indexResult.getResultType()).thenReturn(Engine.Result.Type.SUCCESS); + when(indexResult.getFailure()).thenReturn(new RuntimeException("test exception")); + indexingFailuresDebugListener.postIndex(shardId, index, indexResult); + assertThat(appender.getLastEventAndReset(), nullValue()); + } + +} From b769dcadad31e4438b451952bb17d20ade52c5db Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 7 Dec 2024 00:16:14 +1100 Subject: [PATCH 074/204] Mute org.elasticsearch.packaging.test.KeystoreManagementTests test30KeystorePasswordFromFile #118123 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e4cd94b221536..3609ce846a0cb 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -248,6 +248,9 @@ tests: - class: org.elasticsearch.datastreams.ResolveClusterDataStreamIT method: testClusterResolveWithDataStreamsUsingAlias issue: https://github.com/elastic/elasticsearch/issues/118124 +- class: org.elasticsearch.packaging.test.KeystoreManagementTests + method: test30KeystorePasswordFromFile + issue: https://github.com/elastic/elasticsearch/issues/118123 # Examples: # From 06af8d049adec85ddc4db350bcf6c143fb3c531a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 7 Dec 2024 00:16:29 +1100 Subject: [PATCH 075/204] Mute org.elasticsearch.packaging.test.ArchiveTests test41AutoconfigurationNotTriggeredWhenNodeCannotContainData #118110 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 3609ce846a0cb..c5dfbdcc0623b 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -251,6 +251,9 @@ tests: - class: org.elasticsearch.packaging.test.KeystoreManagementTests method: test30KeystorePasswordFromFile issue: https://github.com/elastic/elasticsearch/issues/118123 +- class: org.elasticsearch.packaging.test.ArchiveTests + method: test41AutoconfigurationNotTriggeredWhenNodeCannotContainData + issue: https://github.com/elastic/elasticsearch/issues/118110 # Examples: # From 2af2d5e5f55883e31447c784cadf50d2213acfef Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Fri, 6 Dec 2024 08:20:34 -0500 Subject: [PATCH 076/204] Unmuting test #116178 (#118100) There was a single valid test failure, and this failure COULD have been caused by some of the weird race conditions introduced around that time in DFS/query phase optimizations. Unmuting to see if its actually fixed Related: https://github.com/elastic/elasticsearch/issues/116178 --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index c5dfbdcc0623b..2b4cc3991a25f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -111,9 +111,6 @@ tests: - class: org.elasticsearch.action.search.SearchPhaseControllerTests method: testProgressListener issue: https://github.com/elastic/elasticsearch/issues/116149 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=terms_enum/10_basic/Test security} - issue: https://github.com/elastic/elasticsearch/issues/116178 - class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT method: testSearchWithRandomDisconnects issue: https://github.com/elastic/elasticsearch/issues/116175 From d42654a6910a2193a0528d7314b2620e8a1a4178 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 6 Dec 2024 07:27:51 -0600 Subject: [PATCH 077/204] Using the credentials of the user who calls reindex data stream (#117938) --- .../ReindexDataStreamTransportAction.java | 4 +- .../task/ExecuteWithHeadersClient.java | 40 ++++++++++ ...indexDataStreamPersistentTaskExecutor.java | 3 +- .../task/ReindexDataStreamTaskParams.java | 50 ++++++++++-- .../ReindexDataStreamTaskParamsTests.java | 79 ++++++++++++++++++- 5 files changed, 163 insertions(+), 13 deletions(-) create mode 100644 x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ExecuteWithHeadersClient.java diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java index d532b001f5aaa..7f68007f821ba 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.ReindexDataStreamRequest; import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.ReindexDataStreamResponse; import org.elasticsearch.xpack.migrate.task.ReindexDataStreamTask; @@ -72,7 +73,8 @@ protected void doExecute(Task task, ReindexDataStreamRequest request, ActionList sourceDataStreamName, transportService.getThreadPool().absoluteTimeInMillis(), totalIndices, - totalIndicesToBeUpgraded + totalIndicesToBeUpgraded, + ClientHelper.getPersistableSafeSecurityHeaders(transportService.getThreadPool().getThreadContext(), clusterService.state()) ); String persistentTaskId = getPersistentTaskId(sourceDataStreamName); persistentTasksService.sendStartRequest( diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ExecuteWithHeadersClient.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ExecuteWithHeadersClient.java new file mode 100644 index 0000000000000..a8962f56468bc --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ExecuteWithHeadersClient.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.task; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.support.AbstractClient; +import org.elasticsearch.xpack.core.ClientHelper; + +import java.util.Map; + +public class ExecuteWithHeadersClient extends AbstractClient { + + private final Client client; + private final Map headers; + + public ExecuteWithHeadersClient(Client client, Map headers) { + super(client.settings(), client.threadPool()); + this.client = client; + this.headers = headers; + } + + @Override + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + ClientHelper.executeWithHeadersAsync(headers, null, client, action, request, listener); + } + +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java index 0f3f8b17f27ad..fc471cfa89f26 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java @@ -66,7 +66,8 @@ protected void nodeOperation(AllocatedPersistentTask task, ReindexDataStreamTask GetDataStreamAction.Request request = new GetDataStreamAction.Request(TimeValue.MAX_VALUE, new String[] { sourceDataStream }); assert task instanceof ReindexDataStreamTask; final ReindexDataStreamTask reindexDataStreamTask = (ReindexDataStreamTask) task; - client.execute(GetDataStreamAction.INSTANCE, request, ActionListener.wrap(response -> { + ExecuteWithHeadersClient reindexClient = new ExecuteWithHeadersClient(client, params.headers()); + reindexClient.execute(GetDataStreamAction.INSTANCE, request, ActionListener.wrap(response -> { List dataStreamInfos = response.getDataStreams(); if (dataStreamInfos.size() == 1) { List indices = dataStreamInfos.getFirst().getDataStream().getIndices(); diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTaskParams.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTaskParams.java index 0f26713a75184..7c4b0007bb632 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTaskParams.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTaskParams.java @@ -9,41 +9,65 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; +import java.util.Map; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -public record ReindexDataStreamTaskParams(String sourceDataStream, long startTime, int totalIndices, int totalIndicesToBeUpgraded) - implements - PersistentTaskParams { +public record ReindexDataStreamTaskParams( + String sourceDataStream, + long startTime, + int totalIndices, + int totalIndicesToBeUpgraded, + Map headers +) implements PersistentTaskParams { + + private static final String API_CONTEXT = Metadata.XContentContext.API.toString(); public static final String NAME = ReindexDataStreamTask.TASK_NAME; private static final String SOURCE_DATA_STREAM_FIELD = "source_data_stream"; private static final String START_TIME_FIELD = "start_time"; private static final String TOTAL_INDICES_FIELD = "total_indices"; private static final String TOTAL_INDICES_TO_BE_UPGRADED_FIELD = "total_indices_to_be_upgraded"; + private static final String HEADERS_FIELD = "headers"; + @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, true, - args -> new ReindexDataStreamTaskParams((String) args[0], (long) args[1], (int) args[2], (int) args[3]) + args -> new ReindexDataStreamTaskParams( + (String) args[0], + (long) args[1], + (int) args[2], + (int) args[3], + args[4] == null ? Map.of() : (Map) args[4] + ) ); static { PARSER.declareString(constructorArg(), new ParseField(SOURCE_DATA_STREAM_FIELD)); PARSER.declareLong(constructorArg(), new ParseField(START_TIME_FIELD)); PARSER.declareInt(constructorArg(), new ParseField(TOTAL_INDICES_FIELD)); PARSER.declareInt(constructorArg(), new ParseField(TOTAL_INDICES_TO_BE_UPGRADED_FIELD)); + PARSER.declareField( + ConstructingObjectParser.optionalConstructorArg(), + XContentParser::mapStrings, + new ParseField(HEADERS_FIELD), + ObjectParser.ValueType.OBJECT + ); } + @SuppressWarnings("unchecked") public ReindexDataStreamTaskParams(StreamInput in) throws IOException { - this(in.readString(), in.readLong(), in.readInt(), in.readInt()); + this(in.readString(), in.readLong(), in.readInt(), in.readInt(), (Map) in.readGenericValue()); } @Override @@ -62,16 +86,22 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(startTime); out.writeInt(totalIndices); out.writeInt(totalIndicesToBeUpgraded); + out.writeGenericValue(headers); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject() + builder.startObject() .field(SOURCE_DATA_STREAM_FIELD, sourceDataStream) .field(START_TIME_FIELD, startTime) .field(TOTAL_INDICES_FIELD, totalIndices) - .field(TOTAL_INDICES_TO_BE_UPGRADED_FIELD, totalIndicesToBeUpgraded) - .endObject(); + .field(TOTAL_INDICES_TO_BE_UPGRADED_FIELD, totalIndicesToBeUpgraded); + if (API_CONTEXT.equals(params.param(Metadata.CONTEXT_MODE_PARAM, API_CONTEXT)) == false) { + // This makes sure that we don't return the headers to an api request, like _cluster/state + builder.stringStringMap(HEADERS_FIELD, headers); + } + builder.endObject(); + return builder; } public String getSourceDataStream() { @@ -81,4 +111,8 @@ public String getSourceDataStream() { public static ReindexDataStreamTaskParams fromXContent(XContentParser parser) { return PARSER.apply(parser, null); } + + public Map getHeaders() { + return headers; + } } diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTaskParamsTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTaskParamsTests.java index fc39b5d8cb703..67ade297f27ad 100644 --- a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTaskParamsTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTaskParamsTests.java @@ -7,11 +7,14 @@ package org.elasticsearch.xpack.migrate.task; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; @@ -29,7 +32,26 @@ protected Writeable.Reader instanceReader() { @Override protected ReindexDataStreamTaskParams createTestInstance() { - return new ReindexDataStreamTaskParams(randomAlphaOfLength(50), randomLong(), randomNonNegativeInt(), randomNonNegativeInt()); + return createTestInstance(randomBoolean()); + } + + @Override + protected ReindexDataStreamTaskParams createXContextTestInstance(XContentType xContentType) { + /* + * Since we filter out headers from xcontent in some cases, we can't use them in the standard xcontent round trip testing. + * Headers are covered in testToXContentContextMode + */ + return createTestInstance(false); + } + + private ReindexDataStreamTaskParams createTestInstance(boolean withHeaders) { + return new ReindexDataStreamTaskParams( + randomAlphaOfLength(50), + randomLong(), + randomNonNegativeInt(), + randomNonNegativeInt(), + getTestHeaders(withHeaders) + ); } @Override @@ -38,14 +60,16 @@ protected ReindexDataStreamTaskParams mutateInstance(ReindexDataStreamTaskParams long startTime = instance.startTime(); int totalIndices = instance.totalIndices(); int totalIndicesToBeUpgraded = instance.totalIndicesToBeUpgraded(); - switch (randomIntBetween(0, 3)) { + Map headers = instance.headers(); + switch (randomIntBetween(0, 4)) { case 0 -> sourceDataStream = randomAlphaOfLength(50); case 1 -> startTime = randomLong(); case 2 -> totalIndices = totalIndices + 1; case 3 -> totalIndices = totalIndicesToBeUpgraded + 1; + case 4 -> headers = headers.isEmpty() ? getTestHeaders(true) : getTestHeaders(); default -> throw new UnsupportedOperationException(); } - return new ReindexDataStreamTaskParams(sourceDataStream, startTime, totalIndices, totalIndicesToBeUpgraded); + return new ReindexDataStreamTaskParams(sourceDataStream, startTime, totalIndices, totalIndicesToBeUpgraded, headers); } @Override @@ -53,6 +77,18 @@ protected ReindexDataStreamTaskParams doParseInstance(XContentParser parser) { return ReindexDataStreamTaskParams.fromXContent(parser); } + private Map getTestHeaders() { + return getTestHeaders(randomBoolean()); + } + + private Map getTestHeaders(boolean nonEmpty) { + if (nonEmpty) { + return Map.of(randomAlphaOfLength(20), randomAlphaOfLength(30)); + } else { + return Map.of(); + } + } + public void testToXContent() throws IOException { ReindexDataStreamTaskParams params = createTestInstance(); try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent)) { @@ -65,4 +101,41 @@ public void testToXContent() throws IOException { } } } + + public void testToXContentContextMode() throws IOException { + ReindexDataStreamTaskParams params = createTestInstance(true); + + // We do not expect to get headers if the "content_mode" is "api" + try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent)) { + builder.humanReadable(true); + ToXContent.Params xContentParams = new ToXContent.MapParams( + Map.of(Metadata.CONTEXT_MODE_PARAM, Metadata.XContentContext.API.toString()) + ); + params.toXContent(builder, xContentParams); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map parserMap = parser.map(); + assertThat(parserMap.get("source_data_stream"), equalTo(params.sourceDataStream())); + assertThat(((Number) parserMap.get("start_time")).longValue(), equalTo(params.startTime())); + assertThat(parserMap.containsKey("headers"), equalTo(false)); + } + } + + // We do expect to get headers if the "content_mode" is anything but "api" + try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent)) { + builder.humanReadable(true); + ToXContent.Params xContentParams = new ToXContent.MapParams( + Map.of( + Metadata.CONTEXT_MODE_PARAM, + randomFrom(Metadata.XContentContext.GATEWAY.toString(), Metadata.XContentContext.SNAPSHOT.toString()) + ) + ); + params.toXContent(builder, xContentParams); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map parserMap = parser.map(); + assertThat(parserMap.get("source_data_stream"), equalTo(params.sourceDataStream())); + assertThat(((Number) parserMap.get("start_time")).longValue(), equalTo(params.startTime())); + assertThat(parserMap.get("headers"), equalTo(params.getHeaders())); + } + } + } } From 07deeee9b3e532e829edfc7a2ed7be6b973de70e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 7 Dec 2024 00:37:20 +1100 Subject: [PATCH 078/204] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT test {lookup-join.LookupMessageFromIndexKeepReordered SYNC} #118150 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 2b4cc3991a25f..de31dc53e7d13 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -251,6 +251,9 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveTests method: test41AutoconfigurationNotTriggeredWhenNodeCannotContainData issue: https://github.com/elastic/elasticsearch/issues/118110 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {lookup-join.LookupMessageFromIndexKeepReordered SYNC} + issue: https://github.com/elastic/elasticsearch/issues/118150 # Examples: # From a827970d0bfefb6313964de5e059d24b6a1d7b12 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 7 Dec 2024 00:37:32 +1100 Subject: [PATCH 079/204] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT test {lookup-join.LookupMessageFromIndexKeepReordered ASYNC} #118151 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index de31dc53e7d13..6d48052fe25e1 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -254,6 +254,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT method: test {lookup-join.LookupMessageFromIndexKeepReordered SYNC} issue: https://github.com/elastic/elasticsearch/issues/118150 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {lookup-join.LookupMessageFromIndexKeepReordered ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/118151 # Examples: # From 54c320ebc9b262e66ab92af660a8a155311059d4 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Fri, 6 Dec 2024 09:07:57 -0500 Subject: [PATCH 080/204] Adding default endpoint for Elastic Rerank (#117939) * Adding default endpoint for Elastic Rerank * CustomElandRerankTaskSettings -> RerankTaskSettings * Update docs/changelog/117939.yaml --- docs/changelog/117939.yaml | 5 ++ .../xpack/inference/DefaultEndPointsIT.java | 40 ++++++++++++++ .../inference/InferenceBaseRestTest.java | 47 ++++++++++++---- .../xpack/inference/InferenceCrudIT.java | 4 +- .../InferenceNamedWriteablesProvider.java | 6 +- .../elasticsearch/CustomElandRerankModel.java | 4 +- .../elasticsearch/ElasticRerankerModel.java | 5 +- .../ElasticsearchInternalService.java | 55 +++++++++++++------ ...kSettings.java => RerankTaskSettings.java} | 25 ++++----- .../ElasticsearchInternalServiceTests.java | 42 +++++++------- ...ests.java => RerankTaskSettingsTests.java} | 48 ++++++++-------- 11 files changed, 185 insertions(+), 96 deletions(-) create mode 100644 docs/changelog/117939.yaml rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/{CustomElandRerankTaskSettings.java => RerankTaskSettings.java} (79%) rename x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/{CustomElandRerankTaskSettingsTests.java => RerankTaskSettingsTests.java} (53%) diff --git a/docs/changelog/117939.yaml b/docs/changelog/117939.yaml new file mode 100644 index 0000000000000..d41111f099f97 --- /dev/null +++ b/docs/changelog/117939.yaml @@ -0,0 +1,5 @@ +pr: 117939 +summary: Adding default endpoint for Elastic Rerank +area: Machine Learning +type: enhancement +issues: [] diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java index ba3e48e11928d..068b3e1f4ce04 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java @@ -57,6 +57,9 @@ public void testGet() throws IOException { var e5Model = getModel(ElasticsearchInternalService.DEFAULT_E5_ID); assertDefaultE5Config(e5Model); + + var rerankModel = getModel(ElasticsearchInternalService.DEFAULT_RERANK_ID); + assertDefaultRerankConfig(rerankModel); } @SuppressWarnings("unchecked") @@ -125,6 +128,42 @@ private static void assertDefaultE5Config(Map modelConfig) { assertDefaultChunkingSettings(modelConfig); } + @SuppressWarnings("unchecked") + public void testInferDeploysDefaultRerank() throws IOException { + var model = getModel(ElasticsearchInternalService.DEFAULT_RERANK_ID); + assertDefaultRerankConfig(model); + + var inputs = List.of("Hello World", "Goodnight moon"); + var query = "but why"; + var queryParams = Map.of("timeout", "120s"); + var results = infer(ElasticsearchInternalService.DEFAULT_RERANK_ID, TaskType.RERANK, inputs, query, queryParams); + var embeddings = (List>) results.get("rerank"); + assertThat(results.toString(), embeddings, hasSize(2)); + } + + @SuppressWarnings("unchecked") + private static void assertDefaultRerankConfig(Map modelConfig) { + assertEquals(modelConfig.toString(), ElasticsearchInternalService.DEFAULT_RERANK_ID, modelConfig.get("inference_id")); + assertEquals(modelConfig.toString(), ElasticsearchInternalService.NAME, modelConfig.get("service")); + assertEquals(modelConfig.toString(), TaskType.RERANK.toString(), modelConfig.get("task_type")); + + var serviceSettings = (Map) modelConfig.get("service_settings"); + assertThat(modelConfig.toString(), serviceSettings.get("model_id"), is(".rerank-v1")); + assertEquals(modelConfig.toString(), 1, serviceSettings.get("num_threads")); + + var adaptiveAllocations = (Map) serviceSettings.get("adaptive_allocations"); + assertThat( + modelConfig.toString(), + adaptiveAllocations, + Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 32)) + ); + + var chunkingSettings = (Map) modelConfig.get("chunking_settings"); + assertNull(chunkingSettings); + var taskSettings = (Map) modelConfig.get("task_settings"); + assertThat(modelConfig.toString(), taskSettings, Matchers.is(Map.of("return_documents", true))); + } + @SuppressWarnings("unchecked") private static void assertDefaultChunkingSettings(Map modelConfig) { var chunkingSettings = (Map) modelConfig.get("chunking_settings"); @@ -159,6 +198,7 @@ public void onFailure(Exception exception) { var request = createInferenceRequest( Strings.format("_inference/%s", ElasticsearchInternalService.DEFAULT_ELSER_ID), inputs, + null, queryParams ); client().performRequestAsync(request, listener); diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java index 4e32ef99d06dd..86c0128a3e53c 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java @@ -333,7 +333,7 @@ private List getInternalAsList(String endpoint) throws IOException { protected Map infer(String modelId, List input) throws IOException { var endpoint = Strings.format("_inference/%s", modelId); - return inferInternal(endpoint, input, Map.of()); + return inferInternal(endpoint, input, null, Map.of()); } protected Deque streamInferOnMockService(String modelId, TaskType taskType, List input) throws Exception { @@ -344,7 +344,7 @@ protected Deque streamInferOnMockService(String modelId, TaskTy private Deque callAsync(String endpoint, List input) throws Exception { var responseConsumer = new AsyncInferenceResponseConsumer(); var request = new Request("POST", endpoint); - request.setJsonEntity(jsonBody(input)); + request.setJsonEntity(jsonBody(input, null)); request.setOptions(RequestOptions.DEFAULT.toBuilder().setHttpAsyncResponseConsumerFactory(() -> responseConsumer).build()); var latch = new CountDownLatch(1); client().performRequestAsync(request, new ResponseListener() { @@ -364,33 +364,60 @@ public void onFailure(Exception exception) { protected Map infer(String modelId, TaskType taskType, List input) throws IOException { var endpoint = Strings.format("_inference/%s/%s", taskType, modelId); - return inferInternal(endpoint, input, Map.of()); + return inferInternal(endpoint, input, null, Map.of()); } protected Map infer(String modelId, TaskType taskType, List input, Map queryParameters) throws IOException { var endpoint = Strings.format("_inference/%s/%s?error_trace", taskType, modelId); - return inferInternal(endpoint, input, queryParameters); + return inferInternal(endpoint, input, null, queryParameters); } - protected Request createInferenceRequest(String endpoint, List input, Map queryParameters) { + protected Map infer( + String modelId, + TaskType taskType, + List input, + String query, + Map queryParameters + ) throws IOException { + var endpoint = Strings.format("_inference/%s/%s?error_trace", taskType, modelId); + return inferInternal(endpoint, input, query, queryParameters); + } + + protected Request createInferenceRequest( + String endpoint, + List input, + @Nullable String query, + Map queryParameters + ) { var request = new Request("POST", endpoint); - request.setJsonEntity(jsonBody(input)); + request.setJsonEntity(jsonBody(input, query)); if (queryParameters.isEmpty() == false) { request.addParameters(queryParameters); } return request; } - private Map inferInternal(String endpoint, List input, Map queryParameters) throws IOException { - var request = createInferenceRequest(endpoint, input, queryParameters); + private Map inferInternal( + String endpoint, + List input, + @Nullable String query, + Map queryParameters + ) throws IOException { + var request = createInferenceRequest(endpoint, input, query, queryParameters); var response = client().performRequest(request); assertOkOrCreated(response); return entityAsMap(response); } - private String jsonBody(List input) { - var bodyBuilder = new StringBuilder("{\"input\": ["); + private String jsonBody(List input, @Nullable String query) { + final StringBuilder bodyBuilder = new StringBuilder("{"); + + if (query != null) { + bodyBuilder.append("\"query\":\"").append(query).append("\","); + } + + bodyBuilder.append("\"input\": ["); for (var in : input) { bodyBuilder.append('"').append(in).append('"').append(','); } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index f5773e73f2b22..604e1d4f553b2 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -44,7 +44,7 @@ public void testCRUD() throws IOException { } var getAllModels = getAllModels(); - int numModels = 11; + int numModels = 12; assertThat(getAllModels, hasSize(numModels)); var getSparseModels = getModels("_all", TaskType.SPARSE_EMBEDDING); @@ -482,7 +482,7 @@ public void testSupportedStream() throws Exception { } public void testGetZeroModels() throws IOException { - var models = getModels("_all", TaskType.RERANK); + var models = getModels("_all", TaskType.COMPLETION); assertThat(models, empty()); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java index 2320cca8295d1..673b841317a3d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java @@ -62,12 +62,12 @@ import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalTextEmbeddingServiceSettings; -import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandRerankTaskSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticRerankerServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElserInternalServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElserMlNodeTaskSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.MultilingualE5SmallInternalServiceSettings; +import org.elasticsearch.xpack.inference.services.elasticsearch.RerankTaskSettings; import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionServiceSettings; import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.googlevertexai.GoogleVertexAiSecretSettings; @@ -510,9 +510,7 @@ private static void addCustomElandWriteables(final List namedWriteables) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java index f620b15680c8d..6388bb33bb78d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java @@ -17,7 +17,7 @@ import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandRerankTaskSettings.RETURN_DOCUMENTS; +import static org.elasticsearch.xpack.inference.services.elasticsearch.RerankTaskSettings.RETURN_DOCUMENTS; public class CustomElandRerankModel extends CustomElandModel { @@ -26,7 +26,7 @@ public CustomElandRerankModel( TaskType taskType, String service, CustomElandInternalServiceSettings serviceSettings, - CustomElandRerankTaskSettings taskSettings + RerankTaskSettings taskSettings ) { super(inferenceEntityId, taskType, service, serviceSettings, taskSettings); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticRerankerModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticRerankerModel.java index 115cc9f05599a..276bce6dbe8f8 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticRerankerModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticRerankerModel.java @@ -9,7 +9,6 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; @@ -22,9 +21,9 @@ public ElasticRerankerModel( TaskType taskType, String service, ElasticRerankerServiceSettings serviceSettings, - ChunkingSettings chunkingSettings + RerankTaskSettings taskSettings ) { - super(inferenceEntityId, taskType, service, serviceSettings, chunkingSettings); + super(inferenceEntityId, taskType, service, serviceSettings, taskSettings); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index 2ec3a9d629434..0e64842f873d3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -101,6 +101,7 @@ public class ElasticsearchInternalService extends BaseElasticsearchInternalServi public static final int EMBEDDING_MAX_BATCH_SIZE = 10; public static final String DEFAULT_ELSER_ID = ".elser-2-elasticsearch"; public static final String DEFAULT_E5_ID = ".multilingual-e5-small-elasticsearch"; + public static final String DEFAULT_RERANK_ID = ".rerank-v1-elasticsearch"; private static final EnumSet supportedTaskTypes = EnumSet.of( TaskType.RERANK, @@ -225,7 +226,7 @@ public void parseRequestConfig( ) ); } else if (RERANKER_ID.equals(modelId)) { - rerankerCase(inferenceEntityId, taskType, config, serviceSettingsMap, chunkingSettings, modelListener); + rerankerCase(inferenceEntityId, taskType, config, serviceSettingsMap, taskSettingsMap, modelListener); } else { customElandCase(inferenceEntityId, taskType, serviceSettingsMap, taskSettingsMap, chunkingSettings, modelListener); } @@ -308,7 +309,7 @@ private static CustomElandModel createCustomElandModel( taskType, NAME, elandServiceSettings(serviceSettings, context), - CustomElandRerankTaskSettings.fromMap(taskSettings) + RerankTaskSettings.fromMap(taskSettings) ); default -> throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST); }; @@ -331,7 +332,7 @@ private void rerankerCase( TaskType taskType, Map config, Map serviceSettingsMap, - ChunkingSettings chunkingSettings, + Map taskSettingsMap, ActionListener modelListener ) { @@ -346,7 +347,7 @@ private void rerankerCase( taskType, NAME, new ElasticRerankerServiceSettings(esServiceSettingsBuilder.build()), - chunkingSettings + RerankTaskSettings.fromMap(taskSettingsMap) ) ); } @@ -512,6 +513,14 @@ public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, M ElserMlNodeTaskSettings.DEFAULT, chunkingSettings ); + } else if (modelId.equals(RERANKER_ID)) { + return new ElasticRerankerModel( + inferenceEntityId, + taskType, + NAME, + new ElasticRerankerServiceSettings(ElasticsearchInternalServiceSettings.fromPersistedMap(serviceSettingsMap)), + RerankTaskSettings.fromMap(taskSettingsMap) + ); } else { return createCustomElandModel( inferenceEntityId, @@ -653,21 +662,23 @@ public void inferRerank( ) { var request = buildInferenceRequest(model.mlNodeDeploymentId(), new TextSimilarityConfigUpdate(query), inputs, inputType, timeout); - var modelSettings = (CustomElandRerankTaskSettings) model.getTaskSettings(); - var requestSettings = CustomElandRerankTaskSettings.fromMap(requestTaskSettings); - Boolean returnDocs = CustomElandRerankTaskSettings.of(modelSettings, requestSettings).returnDocuments(); + var returnDocs = Boolean.TRUE; + if (model.getTaskSettings() instanceof RerankTaskSettings modelSettings) { + var requestSettings = RerankTaskSettings.fromMap(requestTaskSettings); + returnDocs = RerankTaskSettings.of(modelSettings, requestSettings).returnDocuments(); + } Function inputSupplier = returnDocs == Boolean.TRUE ? inputs::get : i -> null; - client.execute( - InferModelAction.INSTANCE, - request, - listener.delegateFailureAndWrap( - (l, inferenceResult) -> l.onResponse( - textSimilarityResultsToRankedDocs(inferenceResult.getInferenceResults(), inputSupplier) - ) - ) + ActionListener mlResultsListener = listener.delegateFailureAndWrap( + (l, inferenceResult) -> l.onResponse(textSimilarityResultsToRankedDocs(inferenceResult.getInferenceResults(), inputSupplier)) + ); + + var maybeDeployListener = mlResultsListener.delegateResponse( + (l, exception) -> maybeStartDeployment(model, exception, request, mlResultsListener) ); + + client.execute(InferModelAction.INSTANCE, request, maybeDeployListener); } public void chunkedInfer( @@ -811,7 +822,8 @@ private RankedDocsResults textSimilarityResultsToRankedDocs( public List defaultConfigIds() { return List.of( new DefaultConfigId(DEFAULT_ELSER_ID, TaskType.SPARSE_EMBEDDING, this), - new DefaultConfigId(DEFAULT_E5_ID, TaskType.TEXT_EMBEDDING, this) + new DefaultConfigId(DEFAULT_E5_ID, TaskType.TEXT_EMBEDDING, this), + new DefaultConfigId(DEFAULT_RERANK_ID, TaskType.RERANK, this) ); } @@ -904,12 +916,19 @@ private List defaultConfigs(boolean useLinuxOptimizedModel) { ), ChunkingSettingsBuilder.DEFAULT_SETTINGS ); - return List.of(defaultElser, defaultE5); + var defaultRerank = new ElasticRerankerModel( + DEFAULT_RERANK_ID, + TaskType.RERANK, + NAME, + new ElasticRerankerServiceSettings(null, 1, RERANKER_ID, new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 32)), + RerankTaskSettings.DEFAULT_SETTINGS + ); + return List.of(defaultElser, defaultE5, defaultRerank); } @Override boolean isDefaultId(String inferenceId) { - return DEFAULT_ELSER_ID.equals(inferenceId) || DEFAULT_E5_ID.equals(inferenceId); + return DEFAULT_ELSER_ID.equals(inferenceId) || DEFAULT_E5_ID.equals(inferenceId) || DEFAULT_RERANK_ID.equals(inferenceId); } static EmbeddingRequestChunker.EmbeddingType embeddingTypeFromTaskTypeAndSettings( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/RerankTaskSettings.java similarity index 79% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/RerankTaskSettings.java index a0be1661b860d..3c25f7a6a9016 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/RerankTaskSettings.java @@ -26,14 +26,14 @@ /** * Defines the task settings for internal rerank service. */ -public class CustomElandRerankTaskSettings implements TaskSettings { +public class RerankTaskSettings implements TaskSettings { public static final String NAME = "custom_eland_rerank_task_settings"; public static final String RETURN_DOCUMENTS = "return_documents"; - static final CustomElandRerankTaskSettings DEFAULT_SETTINGS = new CustomElandRerankTaskSettings(Boolean.TRUE); + static final RerankTaskSettings DEFAULT_SETTINGS = new RerankTaskSettings(Boolean.TRUE); - public static CustomElandRerankTaskSettings defaultsFromMap(Map map) { + public static RerankTaskSettings defaultsFromMap(Map map) { ValidationException validationException = new ValidationException(); if (map == null || map.isEmpty()) { @@ -49,7 +49,7 @@ public static CustomElandRerankTaskSettings defaultsFromMap(Map returnDocuments = true; } - return new CustomElandRerankTaskSettings(returnDocuments); + return new RerankTaskSettings(returnDocuments); } /** @@ -57,13 +57,13 @@ public static CustomElandRerankTaskSettings defaultsFromMap(Map * @param map source map * @return Task settings */ - public static CustomElandRerankTaskSettings fromMap(Map map) { + public static RerankTaskSettings fromMap(Map map) { if (map == null || map.isEmpty()) { return DEFAULT_SETTINGS; } Boolean returnDocuments = extractOptionalBoolean(map, RETURN_DOCUMENTS, new ValidationException()); - return new CustomElandRerankTaskSettings(returnDocuments); + return new RerankTaskSettings(returnDocuments); } /** @@ -74,20 +74,17 @@ public static CustomElandRerankTaskSettings fromMap(Map map) { * @param requestTaskSettings the settings passed in within the task_settings field of the request * @return Either {@code originalSettings} or {@code requestTaskSettings} */ - public static CustomElandRerankTaskSettings of( - CustomElandRerankTaskSettings originalSettings, - CustomElandRerankTaskSettings requestTaskSettings - ) { + public static RerankTaskSettings of(RerankTaskSettings originalSettings, RerankTaskSettings requestTaskSettings) { return requestTaskSettings.returnDocuments() != null ? requestTaskSettings : originalSettings; } private final Boolean returnDocuments; - public CustomElandRerankTaskSettings(StreamInput in) throws IOException { + public RerankTaskSettings(StreamInput in) throws IOException { this(in.readOptionalBoolean()); } - public CustomElandRerankTaskSettings(@Nullable Boolean doReturnDocuments) { + public RerankTaskSettings(@Nullable Boolean doReturnDocuments) { if (doReturnDocuments == null) { this.returnDocuments = true; } else { @@ -133,7 +130,7 @@ public Boolean returnDocuments() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - CustomElandRerankTaskSettings that = (CustomElandRerankTaskSettings) o; + RerankTaskSettings that = (RerankTaskSettings) o; return Objects.equals(returnDocuments, that.returnDocuments); } @@ -144,7 +141,7 @@ public int hashCode() { @Override public TaskSettings updatedTaskSettings(Map newSettings) { - CustomElandRerankTaskSettings updatedSettings = CustomElandRerankTaskSettings.fromMap(new HashMap<>(newSettings)); + RerankTaskSettings updatedSettings = RerankTaskSettings.fromMap(new HashMap<>(newSettings)); return of(this, updatedSettings); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java index 306509ea60cfc..17e6583f11c8f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java @@ -534,16 +534,13 @@ public void testParseRequestConfig_Rerank() { ) ); var returnDocs = randomBoolean(); - settings.put( - ModelConfigurations.TASK_SETTINGS, - new HashMap<>(Map.of(CustomElandRerankTaskSettings.RETURN_DOCUMENTS, returnDocs)) - ); + settings.put(ModelConfigurations.TASK_SETTINGS, new HashMap<>(Map.of(RerankTaskSettings.RETURN_DOCUMENTS, returnDocs))); ActionListener modelListener = ActionListener.wrap(model -> { assertThat(model, instanceOf(CustomElandRerankModel.class)); - assertThat(model.getTaskSettings(), instanceOf(CustomElandRerankTaskSettings.class)); + assertThat(model.getTaskSettings(), instanceOf(RerankTaskSettings.class)); assertThat(model.getServiceSettings(), instanceOf(CustomElandInternalServiceSettings.class)); - assertEquals(returnDocs, ((CustomElandRerankTaskSettings) model.getTaskSettings()).returnDocuments()); + assertEquals(returnDocs, ((RerankTaskSettings) model.getTaskSettings()).returnDocuments()); }, e -> { fail("Model parsing failed " + e.getMessage()); }); service.parseRequestConfig(randomInferenceEntityId, TaskType.RERANK, settings, modelListener); @@ -583,9 +580,9 @@ public void testParseRequestConfig_Rerank_DefaultTaskSettings() { ActionListener modelListener = ActionListener.wrap(model -> { assertThat(model, instanceOf(CustomElandRerankModel.class)); - assertThat(model.getTaskSettings(), instanceOf(CustomElandRerankTaskSettings.class)); + assertThat(model.getTaskSettings(), instanceOf(RerankTaskSettings.class)); assertThat(model.getServiceSettings(), instanceOf(CustomElandInternalServiceSettings.class)); - assertEquals(Boolean.TRUE, ((CustomElandRerankTaskSettings) model.getTaskSettings()).returnDocuments()); + assertEquals(Boolean.TRUE, ((RerankTaskSettings) model.getTaskSettings()).returnDocuments()); }, e -> { fail("Model parsing failed " + e.getMessage()); }); service.parseRequestConfig(randomInferenceEntityId, TaskType.RERANK, settings, modelListener); @@ -1249,14 +1246,11 @@ public void testParsePersistedConfig_Rerank() { ); settings.put(ElasticsearchInternalServiceSettings.MODEL_ID, "foo"); var returnDocs = randomBoolean(); - settings.put( - ModelConfigurations.TASK_SETTINGS, - new HashMap<>(Map.of(CustomElandRerankTaskSettings.RETURN_DOCUMENTS, returnDocs)) - ); + settings.put(ModelConfigurations.TASK_SETTINGS, new HashMap<>(Map.of(RerankTaskSettings.RETURN_DOCUMENTS, returnDocs))); var model = service.parsePersistedConfig(randomInferenceEntityId, TaskType.RERANK, settings); - assertThat(model.getTaskSettings(), instanceOf(CustomElandRerankTaskSettings.class)); - assertEquals(returnDocs, ((CustomElandRerankTaskSettings) model.getTaskSettings()).returnDocuments()); + assertThat(model.getTaskSettings(), instanceOf(RerankTaskSettings.class)); + assertEquals(returnDocs, ((RerankTaskSettings) model.getTaskSettings()).returnDocuments()); } // without task settings @@ -1279,8 +1273,8 @@ public void testParsePersistedConfig_Rerank() { settings.put(ElasticsearchInternalServiceSettings.MODEL_ID, "foo"); var model = service.parsePersistedConfig(randomInferenceEntityId, TaskType.RERANK, settings); - assertThat(model.getTaskSettings(), instanceOf(CustomElandRerankTaskSettings.class)); - assertTrue(((CustomElandRerankTaskSettings) model.getTaskSettings()).returnDocuments()); + assertThat(model.getTaskSettings(), instanceOf(RerankTaskSettings.class)); + assertTrue(((RerankTaskSettings) model.getTaskSettings()).returnDocuments()); } } @@ -1335,7 +1329,7 @@ private CustomElandModel getCustomElandModel(TaskType taskType) { taskType, ElasticsearchInternalService.NAME, new CustomElandInternalServiceSettings(1, 4, "custom-model", null), - CustomElandRerankTaskSettings.DEFAULT_SETTINGS + RerankTaskSettings.DEFAULT_SETTINGS ); } else if (taskType == TaskType.TEXT_EMBEDDING) { var serviceSettings = new CustomElandInternalTextEmbeddingServiceSettings(1, 4, "custom-model", null); @@ -1528,20 +1522,30 @@ public void testEmbeddingTypeFromTaskTypeAndSettings() { ) ); - var e = expectThrows( + var e1 = expectThrows( ElasticsearchStatusException.class, () -> ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( TaskType.COMPLETION, new ElasticsearchInternalServiceSettings(1, 1, "foo", null) ) ); - assertThat(e.getMessage(), containsString("Chunking is not supported for task type [completion]")); + assertThat(e1.getMessage(), containsString("Chunking is not supported for task type [completion]")); + + var e2 = expectThrows( + ElasticsearchStatusException.class, + () -> ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( + TaskType.RERANK, + new ElasticsearchInternalServiceSettings(1, 1, "foo", null) + ) + ); + assertThat(e2.getMessage(), containsString("Chunking is not supported for task type [rerank]")); } public void testIsDefaultId() { var service = createService(mock(Client.class)); assertTrue(service.isDefaultId(".elser-2-elasticsearch")); assertTrue(service.isDefaultId(".multilingual-e5-small-elasticsearch")); + assertTrue(service.isDefaultId(".rerank-v1-elasticsearch")); assertFalse(service.isDefaultId("foo")); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/RerankTaskSettingsTests.java similarity index 53% rename from x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java rename to x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/RerankTaskSettingsTests.java index 4207896fc54f3..255454a1ed62b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/RerankTaskSettingsTests.java @@ -22,7 +22,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.sameInstance; -public class CustomElandRerankTaskSettingsTests extends AbstractWireSerializingTestCase { +public class RerankTaskSettingsTests extends AbstractWireSerializingTestCase { public void testIsEmpty() { var randomSettings = createRandom(); @@ -35,9 +35,9 @@ public void testUpdatedTaskSettings() { var newSettings = createRandom(); Map newSettingsMap = new HashMap<>(); if (newSettings.returnDocuments() != null) { - newSettingsMap.put(CustomElandRerankTaskSettings.RETURN_DOCUMENTS, newSettings.returnDocuments()); + newSettingsMap.put(RerankTaskSettings.RETURN_DOCUMENTS, newSettings.returnDocuments()); } - CustomElandRerankTaskSettings updatedSettings = (CustomElandRerankTaskSettings) initialSettings.updatedTaskSettings( + RerankTaskSettings updatedSettings = (RerankTaskSettings) initialSettings.updatedTaskSettings( Collections.unmodifiableMap(newSettingsMap) ); if (newSettings.returnDocuments() == null) { @@ -48,37 +48,37 @@ public void testUpdatedTaskSettings() { } public void testDefaultsFromMap_MapIsNull_ReturnsDefaultSettings() { - var customElandRerankTaskSettings = CustomElandRerankTaskSettings.defaultsFromMap(null); + var rerankTaskSettings = RerankTaskSettings.defaultsFromMap(null); - assertThat(customElandRerankTaskSettings, sameInstance(CustomElandRerankTaskSettings.DEFAULT_SETTINGS)); + assertThat(rerankTaskSettings, sameInstance(RerankTaskSettings.DEFAULT_SETTINGS)); } public void testDefaultsFromMap_MapIsEmpty_ReturnsDefaultSettings() { - var customElandRerankTaskSettings = CustomElandRerankTaskSettings.defaultsFromMap(new HashMap<>()); + var rerankTaskSettings = RerankTaskSettings.defaultsFromMap(new HashMap<>()); - assertThat(customElandRerankTaskSettings, sameInstance(CustomElandRerankTaskSettings.DEFAULT_SETTINGS)); + assertThat(rerankTaskSettings, sameInstance(RerankTaskSettings.DEFAULT_SETTINGS)); } public void testDefaultsFromMap_ExtractedReturnDocumentsNull_SetsReturnDocumentToTrue() { - var customElandRerankTaskSettings = CustomElandRerankTaskSettings.defaultsFromMap(new HashMap<>()); + var rerankTaskSettings = RerankTaskSettings.defaultsFromMap(new HashMap<>()); - assertThat(customElandRerankTaskSettings.returnDocuments(), is(Boolean.TRUE)); + assertThat(rerankTaskSettings.returnDocuments(), is(Boolean.TRUE)); } public void testFromMap_MapIsNull_ReturnsDefaultSettings() { - var customElandRerankTaskSettings = CustomElandRerankTaskSettings.fromMap(null); + var rerankTaskSettings = RerankTaskSettings.fromMap(null); - assertThat(customElandRerankTaskSettings, sameInstance(CustomElandRerankTaskSettings.DEFAULT_SETTINGS)); + assertThat(rerankTaskSettings, sameInstance(RerankTaskSettings.DEFAULT_SETTINGS)); } public void testFromMap_MapIsEmpty_ReturnsDefaultSettings() { - var customElandRerankTaskSettings = CustomElandRerankTaskSettings.fromMap(new HashMap<>()); + var rerankTaskSettings = RerankTaskSettings.fromMap(new HashMap<>()); - assertThat(customElandRerankTaskSettings, sameInstance(CustomElandRerankTaskSettings.DEFAULT_SETTINGS)); + assertThat(rerankTaskSettings, sameInstance(RerankTaskSettings.DEFAULT_SETTINGS)); } public void testToXContent_WritesAllValues() throws IOException { - var serviceSettings = new CustomElandRerankTaskSettings(Boolean.TRUE); + var serviceSettings = new RerankTaskSettings(Boolean.TRUE); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); serviceSettings.toXContent(builder, null); @@ -89,30 +89,30 @@ public void testToXContent_WritesAllValues() throws IOException { } public void testOf_PrefersNonNullRequestTaskSettings() { - var originalSettings = new CustomElandRerankTaskSettings(Boolean.FALSE); - var requestTaskSettings = new CustomElandRerankTaskSettings(Boolean.TRUE); + var originalSettings = new RerankTaskSettings(Boolean.FALSE); + var requestTaskSettings = new RerankTaskSettings(Boolean.TRUE); - var taskSettings = CustomElandRerankTaskSettings.of(originalSettings, requestTaskSettings); + var taskSettings = RerankTaskSettings.of(originalSettings, requestTaskSettings); assertThat(taskSettings, sameInstance(requestTaskSettings)); } - private static CustomElandRerankTaskSettings createRandom() { - return new CustomElandRerankTaskSettings(randomOptionalBoolean()); + private static RerankTaskSettings createRandom() { + return new RerankTaskSettings(randomOptionalBoolean()); } @Override - protected Writeable.Reader instanceReader() { - return CustomElandRerankTaskSettings::new; + protected Writeable.Reader instanceReader() { + return RerankTaskSettings::new; } @Override - protected CustomElandRerankTaskSettings createTestInstance() { + protected RerankTaskSettings createTestInstance() { return createRandom(); } @Override - protected CustomElandRerankTaskSettings mutateInstance(CustomElandRerankTaskSettings instance) throws IOException { - return randomValueOtherThan(instance, CustomElandRerankTaskSettingsTests::createRandom); + protected RerankTaskSettings mutateInstance(RerankTaskSettings instance) throws IOException { + return randomValueOtherThan(instance, RerankTaskSettingsTests::createRandom); } } From 7cd17d21856f0a90b5eb2bfbe6fa37cc3f5d3ce1 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Fri, 6 Dec 2024 09:17:32 -0500 Subject: [PATCH 081/204] Esql compare nanos and millis (#118027) Resolves #116281 Introduces support for comparing millisecond dates with nanosecond dates, without the need for casting. Millisecond dates outside of the nanosecond date range are handled correctly. --- docs/changelog/118027.yaml | 6 + .../functions/kibana/definition/equals.json | 36 +++++ .../kibana/definition/greater_than.json | 36 +++++ .../definition/greater_than_or_equal.json | 36 +++++ .../kibana/definition/less_than.json | 36 +++++ .../kibana/definition/less_than_or_equal.json | 36 +++++ .../kibana/definition/not_equals.json | 36 +++++ .../esql/functions/types/equals.asciidoc | 2 + .../functions/types/greater_than.asciidoc | 2 + .../types/greater_than_or_equal.asciidoc | 2 + .../esql/functions/types/less_than.asciidoc | 2 + .../types/less_than_or_equal.asciidoc | 2 + .../esql/functions/types/not_equals.asciidoc | 2 + .../elasticsearch/common/time/DateUtils.java | 31 ++++ .../common/time/DateUtilsTests.java | 40 +++++ .../xpack/esql/EsqlTestUtils.java | 2 +- .../src/main/resources/date_nanos.csv-spec | 95 ++++++++++- .../EqualsMillisNanosEvaluator.java | 148 ++++++++++++++++++ .../EqualsNanosMillisEvaluator.java | 148 ++++++++++++++++++ .../GreaterThanMillisNanosEvaluator.java | 148 ++++++++++++++++++ .../GreaterThanNanosMillisEvaluator.java | 148 ++++++++++++++++++ ...reaterThanOrEqualMillisNanosEvaluator.java | 148 ++++++++++++++++++ ...reaterThanOrEqualNanosMillisEvaluator.java | 148 ++++++++++++++++++ .../LessThanMillisNanosEvaluator.java | 148 ++++++++++++++++++ .../LessThanNanosMillisEvaluator.java | 148 ++++++++++++++++++ .../LessThanOrEqualMillisNanosEvaluator.java | 148 ++++++++++++++++++ .../LessThanOrEqualNanosMillisEvaluator.java | 148 ++++++++++++++++++ .../NotEqualsMillisNanosEvaluator.java | 148 ++++++++++++++++++ .../NotEqualsNanosMillisEvaluator.java | 148 ++++++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../xpack/esql/analysis/Verifier.java | 13 +- .../predicate/operator/comparison/Equals.java | 32 +++- .../comparison/EsqlBinaryComparison.java | 36 ++++- .../operator/comparison/GreaterThan.java | 33 +++- .../comparison/GreaterThanOrEqual.java | 33 +++- .../operator/comparison/LessThan.java | 23 ++- .../operator/comparison/LessThanOrEqual.java | 23 ++- .../operator/comparison/NotEquals.java | 32 +++- .../expression/function/TestCaseSupplier.java | 4 +- .../operator/comparison/EqualsTests.java | 28 ++++ .../comparison/GreaterThanOrEqualTests.java | 28 ++++ .../operator/comparison/GreaterThanTests.java | 28 ++++ .../comparison/LessThanOrEqualTests.java | 28 ++++ .../operator/comparison/LessThanTests.java | 28 ++++ .../operator/comparison/NotEqualsTests.java | 32 +++- 45 files changed, 2560 insertions(+), 24 deletions(-) create mode 100644 docs/changelog/118027.yaml create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsMillisNanosEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsNanosMillisEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanMillisNanosEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanNanosMillisEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualMillisNanosEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualNanosMillisEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanMillisNanosEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanNanosMillisEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualMillisNanosEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualNanosMillisEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsMillisNanosEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsNanosMillisEvaluator.java diff --git a/docs/changelog/118027.yaml b/docs/changelog/118027.yaml new file mode 100644 index 0000000000000..161c156b56a65 --- /dev/null +++ b/docs/changelog/118027.yaml @@ -0,0 +1,6 @@ +pr: 118027 +summary: Esql compare nanos and millis +area: ES|QL +type: enhancement +issues: + - 116281 diff --git a/docs/reference/esql/functions/kibana/definition/equals.json b/docs/reference/esql/functions/kibana/definition/equals.json index 885d949f4b20f..40f3d54ba597a 100644 --- a/docs/reference/esql/functions/kibana/definition/equals.json +++ b/docs/reference/esql/functions/kibana/definition/equals.json @@ -77,6 +77,42 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/greater_than.json b/docs/reference/esql/functions/kibana/definition/greater_than.json index cf6e30a0a4547..ea2c0fb1212c7 100644 --- a/docs/reference/esql/functions/kibana/definition/greater_than.json +++ b/docs/reference/esql/functions/kibana/definition/greater_than.json @@ -23,6 +23,42 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json b/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json index 2535c68af6acf..7e1feb37e87b0 100644 --- a/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json +++ b/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json @@ -23,6 +23,42 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/less_than.json b/docs/reference/esql/functions/kibana/definition/less_than.json index a73754d200d46..71aae4d759ecf 100644 --- a/docs/reference/esql/functions/kibana/definition/less_than.json +++ b/docs/reference/esql/functions/kibana/definition/less_than.json @@ -23,6 +23,42 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json b/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json index 7af477db32a34..f119b7ab2eb12 100644 --- a/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json +++ b/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json @@ -23,6 +23,42 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/not_equals.json b/docs/reference/esql/functions/kibana/definition/not_equals.json index 24f31115cbc37..d35a5b43ec238 100644 --- a/docs/reference/esql/functions/kibana/definition/not_equals.json +++ b/docs/reference/esql/functions/kibana/definition/not_equals.json @@ -77,6 +77,42 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/types/equals.asciidoc b/docs/reference/esql/functions/types/equals.asciidoc index 8d48b7ebf084a..1bb8bf2122b35 100644 --- a/docs/reference/esql/functions/types/equals.asciidoc +++ b/docs/reference/esql/functions/types/equals.asciidoc @@ -9,6 +9,8 @@ boolean | boolean | boolean cartesian_point | cartesian_point | boolean cartesian_shape | cartesian_shape | boolean date | date | boolean +date | date_nanos | boolean +date_nanos | date | boolean date_nanos | date_nanos | boolean double | double | boolean double | integer | boolean diff --git a/docs/reference/esql/functions/types/greater_than.asciidoc b/docs/reference/esql/functions/types/greater_than.asciidoc index 8000fd34c8507..39253ac445f42 100644 --- a/docs/reference/esql/functions/types/greater_than.asciidoc +++ b/docs/reference/esql/functions/types/greater_than.asciidoc @@ -6,6 +6,8 @@ |=== lhs | rhs | result date | date | boolean +date | date_nanos | boolean +date_nanos | date | boolean date_nanos | date_nanos | boolean double | double | boolean double | integer | boolean diff --git a/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc b/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc index 8000fd34c8507..39253ac445f42 100644 --- a/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc +++ b/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc @@ -6,6 +6,8 @@ |=== lhs | rhs | result date | date | boolean +date | date_nanos | boolean +date_nanos | date | boolean date_nanos | date_nanos | boolean double | double | boolean double | integer | boolean diff --git a/docs/reference/esql/functions/types/less_than.asciidoc b/docs/reference/esql/functions/types/less_than.asciidoc index 8000fd34c8507..39253ac445f42 100644 --- a/docs/reference/esql/functions/types/less_than.asciidoc +++ b/docs/reference/esql/functions/types/less_than.asciidoc @@ -6,6 +6,8 @@ |=== lhs | rhs | result date | date | boolean +date | date_nanos | boolean +date_nanos | date | boolean date_nanos | date_nanos | boolean double | double | boolean double | integer | boolean diff --git a/docs/reference/esql/functions/types/less_than_or_equal.asciidoc b/docs/reference/esql/functions/types/less_than_or_equal.asciidoc index 8000fd34c8507..39253ac445f42 100644 --- a/docs/reference/esql/functions/types/less_than_or_equal.asciidoc +++ b/docs/reference/esql/functions/types/less_than_or_equal.asciidoc @@ -6,6 +6,8 @@ |=== lhs | rhs | result date | date | boolean +date | date_nanos | boolean +date_nanos | date | boolean date_nanos | date_nanos | boolean double | double | boolean double | integer | boolean diff --git a/docs/reference/esql/functions/types/not_equals.asciidoc b/docs/reference/esql/functions/types/not_equals.asciidoc index 8d48b7ebf084a..1bb8bf2122b35 100644 --- a/docs/reference/esql/functions/types/not_equals.asciidoc +++ b/docs/reference/esql/functions/types/not_equals.asciidoc @@ -9,6 +9,8 @@ boolean | boolean | boolean cartesian_point | cartesian_point | boolean cartesian_shape | cartesian_shape | boolean date | date | boolean +date | date_nanos | boolean +date_nanos | date | boolean date_nanos | date_nanos | boolean double | double | boolean double | integer | boolean diff --git a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java index 9f642734ba832..72306b6ed675e 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java @@ -293,6 +293,37 @@ public static long toMilliSeconds(long nanoSecondsSinceEpoch) { return nanoSecondsSinceEpoch / 1_000_000; } + /** + * Compare an epoch nanosecond date (such as returned by {@link DateUtils#toLong} + * to an epoch millisecond date (such as returned by {@link Instant#toEpochMilli()}}. + *

+ * NB: This function does not implement {@link java.util.Comparator} in + * order to avoid performance costs of autoboxing the input longs. + * + * @param nanos Epoch date represented as a long number of nanoseconds. + * Note that Elasticsearch does not support nanosecond dates + * before Epoch, so this number should never be negative. + * @param millis Epoch date represented as a long number of milliseconds. + * This parameter does not have to be constrained to the + * range of long nanosecond dates. + * @return -1 if the nanosecond date is before the millisecond date, + * 0 if the two dates represent the same instant, + * 1 if the nanosecond date is after the millisecond date + */ + public static int compareNanosToMillis(long nanos, long millis) { + assert nanos >= 0; + if (millis < 0) { + return 1; + } + if (millis > MAX_NANOSECOND_IN_MILLIS) { + return -1; + } + // This can't overflow, because we know millis is between 0 and MAX_NANOSECOND_IN_MILLIS, + // and MAX_NANOSECOND_IN_MILLIS * 1_000_000 doesn't overflow. + long diff = nanos - (millis * 1_000_000); + return diff == 0 ? 0 : diff < 0 ? -1 : 1; + } + /** * Rounds the given utc milliseconds sicne the epoch down to the next unit millis * diff --git a/server/src/test/java/org/elasticsearch/common/time/DateUtilsTests.java b/server/src/test/java/org/elasticsearch/common/time/DateUtilsTests.java index 2dd0a28013058..e15bbbf75a529 100644 --- a/server/src/test/java/org/elasticsearch/common/time/DateUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/common/time/DateUtilsTests.java @@ -20,7 +20,11 @@ import java.time.ZonedDateTime; import java.time.temporal.ChronoField; +import static org.elasticsearch.common.time.DateUtils.MAX_MILLIS_BEFORE_MINUS_9999; +import static org.elasticsearch.common.time.DateUtils.MAX_NANOSECOND_INSTANT; +import static org.elasticsearch.common.time.DateUtils.MAX_NANOSECOND_IN_MILLIS; import static org.elasticsearch.common.time.DateUtils.clampToNanosRange; +import static org.elasticsearch.common.time.DateUtils.compareNanosToMillis; import static org.elasticsearch.common.time.DateUtils.toInstant; import static org.elasticsearch.common.time.DateUtils.toLong; import static org.elasticsearch.common.time.DateUtils.toMilliSeconds; @@ -28,9 +32,45 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; public class DateUtilsTests extends ESTestCase { + public void testCompareNanosToMillis() { + assertThat(MAX_NANOSECOND_IN_MILLIS * 1_000_000, lessThan(Long.MAX_VALUE)); + + assertThat(compareNanosToMillis(toLong(Instant.EPOCH), Instant.EPOCH.toEpochMilli()), is(0)); + + // This should be 1, because the millisecond version should truncate a bit + assertThat(compareNanosToMillis(toLong(MAX_NANOSECOND_INSTANT), MAX_NANOSECOND_INSTANT.toEpochMilli()), is(1)); + + assertThat(compareNanosToMillis(toLong(MAX_NANOSECOND_INSTANT), -1000), is(1)); + // millis before epoch + assertCompareInstants( + randomInstantBetween(Instant.EPOCH, MAX_NANOSECOND_INSTANT), + randomInstantBetween(Instant.ofEpochMilli(MAX_MILLIS_BEFORE_MINUS_9999), Instant.ofEpochMilli(-1L)) + ); + + // millis after nanos range + assertCompareInstants( + randomInstantBetween(Instant.EPOCH, MAX_NANOSECOND_INSTANT), + randomInstantBetween(MAX_NANOSECOND_INSTANT.plusMillis(1), Instant.ofEpochMilli(Long.MAX_VALUE)) + ); + + // both in range + Instant nanos = randomInstantBetween(Instant.EPOCH, MAX_NANOSECOND_INSTANT); + Instant millis = randomInstantBetween(Instant.EPOCH, MAX_NANOSECOND_INSTANT); + + assertCompareInstants(nanos, millis); + } + + /** + * check that compareNanosToMillis is consistent with Instant#compare. + */ + private void assertCompareInstants(Instant nanos, Instant millis) { + assertThat(compareNanosToMillis(toLong(nanos), millis.toEpochMilli()), equalTo(nanos.compareTo(millis))); + } + public void testInstantToLong() { assertThat(toLong(Instant.EPOCH), is(0L)); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index d6715a932c075..ec9af33dd6690 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -726,7 +726,7 @@ public static Literal randomLiteral(DataType type) { case UNSIGNED_LONG, LONG, COUNTER_LONG -> randomLong(); case DATE_PERIOD -> Period.of(randomIntBetween(-1000, 1000), randomIntBetween(-13, 13), randomIntBetween(-32, 32)); case DATETIME -> randomMillisUpToYear9999(); - case DATE_NANOS -> randomLong(); + case DATE_NANOS -> randomLongBetween(0, Long.MAX_VALUE); case DOUBLE, SCALED_FLOAT, COUNTER_DOUBLE -> randomDouble(); case FLOAT -> randomFloat(); case HALF_FLOAT -> HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat())); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index daa45825b93fc..0d113c0422562 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -216,11 +216,40 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 ; +date nanos greater than millis +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis + +FROM date_nanos | WHERE MV_MIN(nanos) > TO_DATETIME("2023-10-23T12:27:28.948Z") | SORT nanos DESC; + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654 +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878 +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837 +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 +; + date nanos greater than or equal required_capability: to_date_nanos required_capability: date_nanos_binary_comparison -FROM date_nanos | WHERE MV_MIN(nanos) >= TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z") | SORT nanos DESC; +FROM date_nanos | WHERE MV_MIN(nanos) >= TO_DATE_NANOS("2023-10-23T12:27:28.948Z") | SORT nanos DESC; + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654 +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878 +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837 +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + +date nanos greater than or equal millis +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis + +FROM date_nanos | WHERE MV_MIN(nanos) >= TO_DATETIME("2023-10-23T12:27:28.948Z") | SORT nanos DESC; millis:date | nanos:date_nanos | num:long 2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 @@ -231,11 +260,23 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 ; + date nanos less than required_capability: to_date_nanos required_capability: date_nanos_binary_comparison -FROM date_nanos | WHERE MV_MIN(nanos) < TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z") AND millis > "2000-01-01" | SORT nanos DESC; +FROM date_nanos | WHERE MV_MIN(nanos) < TO_DATE_NANOS("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC; + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +; + +date nanos less than millis +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis + +FROM date_nanos | WHERE MV_MIN(nanos) < TO_DATETIME("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC; millis:date | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 @@ -246,7 +287,19 @@ date nanos less than equal required_capability: to_date_nanos required_capability: date_nanos_binary_comparison -FROM date_nanos | WHERE MV_MIN(nanos) <= TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z") AND millis > "2000-01-01" | SORT nanos DESC; +FROM date_nanos | WHERE MV_MIN(nanos) <= TO_DATE_NANOS("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC; + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +; + +date nanos less than equal millis +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis + +FROM date_nanos | WHERE MV_MIN(nanos) <= TO_DATETIME("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC; millis:date | nanos:date_nanos | num:long 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 @@ -254,6 +307,7 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 ; + date nanos equals required_capability: to_date_nanos required_capability: date_nanos_binary_comparison @@ -264,6 +318,25 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 ; +date nanos equals millis exact match +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis + +FROM date_nanos | WHERE MV_MIN(nanos) == TO_DATETIME("2023-10-23T12:27:28.948Z"); + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + +date nanos equals millis without exact match +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis + +FROM date_nanos | WHERE MV_MIN(nanos) == TO_DATETIME("2023-10-23T13:33:34.937"); + +millis:date | nanos:date_nanos | num:long +; + date nanos not equals required_capability: to_date_nanos required_capability: date_nanos_binary_comparison @@ -280,6 +353,22 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 ; +date nanos not equals millis +required_capability: date_nanos_type +required_capability: date_nanos_compare_to_millis + +FROM date_nanos | WHERE MV_MIN(nanos) != TO_DATETIME("2023-10-23T12:27:28.948Z") AND millis > "2000-01-01" | SORT nanos DESC; + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654 +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878 +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837 +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +; + date nanos to long, index version required_capability: to_date_nanos diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsMillisNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsMillisNanosEvaluator.java new file mode 100644 index 0000000000000..b5013c4080507 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsMillisNanosEvaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Equals}. + * This class is generated. Do not edit it. + */ +public final class EqualsMillisNanosEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public EqualsMillisNanosEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + LongVector rhsVector = rhsBlock.asVector(); + if (rhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock rhsBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rhsBlock.getValueCount(p) != 1) { + if (rhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBoolean(Equals.processMillisNanos(lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsBlock.getLong(rhsBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, LongVector lhsVector, LongVector rhsVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBoolean(p, Equals.processMillisNanos(lhsVector.getLong(p), rhsVector.getLong(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "EqualsMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, rhs); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory lhs; + + private final EvalOperator.ExpressionEvaluator.Factory rhs; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, + EvalOperator.ExpressionEvaluator.Factory rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public EqualsMillisNanosEvaluator get(DriverContext context) { + return new EqualsMillisNanosEvaluator(source, lhs.get(context), rhs.get(context), context); + } + + @Override + public String toString() { + return "EqualsMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsNanosMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsNanosMillisEvaluator.java new file mode 100644 index 0000000000000..3ed1e922608e6 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsNanosMillisEvaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Equals}. + * This class is generated. Do not edit it. + */ +public final class EqualsNanosMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public EqualsNanosMillisEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + LongVector rhsVector = rhsBlock.asVector(); + if (rhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock rhsBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rhsBlock.getValueCount(p) != 1) { + if (rhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBoolean(Equals.processNanosMillis(lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsBlock.getLong(rhsBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, LongVector lhsVector, LongVector rhsVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBoolean(p, Equals.processNanosMillis(lhsVector.getLong(p), rhsVector.getLong(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "EqualsNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, rhs); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory lhs; + + private final EvalOperator.ExpressionEvaluator.Factory rhs; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, + EvalOperator.ExpressionEvaluator.Factory rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public EqualsNanosMillisEvaluator get(DriverContext context) { + return new EqualsNanosMillisEvaluator(source, lhs.get(context), rhs.get(context), context); + } + + @Override + public String toString() { + return "EqualsNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanMillisNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanMillisNanosEvaluator.java new file mode 100644 index 0000000000000..bdd877c7f866e --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanMillisNanosEvaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link GreaterThan}. + * This class is generated. Do not edit it. + */ +public final class GreaterThanMillisNanosEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public GreaterThanMillisNanosEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + LongVector rhsVector = rhsBlock.asVector(); + if (rhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock rhsBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rhsBlock.getValueCount(p) != 1) { + if (rhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBoolean(GreaterThan.processMillisNanos(lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsBlock.getLong(rhsBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, LongVector lhsVector, LongVector rhsVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBoolean(p, GreaterThan.processMillisNanos(lhsVector.getLong(p), rhsVector.getLong(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "GreaterThanMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, rhs); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory lhs; + + private final EvalOperator.ExpressionEvaluator.Factory rhs; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, + EvalOperator.ExpressionEvaluator.Factory rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public GreaterThanMillisNanosEvaluator get(DriverContext context) { + return new GreaterThanMillisNanosEvaluator(source, lhs.get(context), rhs.get(context), context); + } + + @Override + public String toString() { + return "GreaterThanMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanNanosMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanNanosMillisEvaluator.java new file mode 100644 index 0000000000000..d509547eb17ce --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanNanosMillisEvaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link GreaterThan}. + * This class is generated. Do not edit it. + */ +public final class GreaterThanNanosMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public GreaterThanNanosMillisEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + LongVector rhsVector = rhsBlock.asVector(); + if (rhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock rhsBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rhsBlock.getValueCount(p) != 1) { + if (rhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBoolean(GreaterThan.processNanosMillis(lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsBlock.getLong(rhsBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, LongVector lhsVector, LongVector rhsVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBoolean(p, GreaterThan.processNanosMillis(lhsVector.getLong(p), rhsVector.getLong(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "GreaterThanNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, rhs); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory lhs; + + private final EvalOperator.ExpressionEvaluator.Factory rhs; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, + EvalOperator.ExpressionEvaluator.Factory rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public GreaterThanNanosMillisEvaluator get(DriverContext context) { + return new GreaterThanNanosMillisEvaluator(source, lhs.get(context), rhs.get(context), context); + } + + @Override + public String toString() { + return "GreaterThanNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualMillisNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualMillisNanosEvaluator.java new file mode 100644 index 0000000000000..7a0da0a55d0dc --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualMillisNanosEvaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link GreaterThanOrEqual}. + * This class is generated. Do not edit it. + */ +public final class GreaterThanOrEqualMillisNanosEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public GreaterThanOrEqualMillisNanosEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + LongVector rhsVector = rhsBlock.asVector(); + if (rhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock rhsBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rhsBlock.getValueCount(p) != 1) { + if (rhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBoolean(GreaterThanOrEqual.processMillisNanos(lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsBlock.getLong(rhsBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, LongVector lhsVector, LongVector rhsVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBoolean(p, GreaterThanOrEqual.processMillisNanos(lhsVector.getLong(p), rhsVector.getLong(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "GreaterThanOrEqualMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, rhs); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory lhs; + + private final EvalOperator.ExpressionEvaluator.Factory rhs; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, + EvalOperator.ExpressionEvaluator.Factory rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public GreaterThanOrEqualMillisNanosEvaluator get(DriverContext context) { + return new GreaterThanOrEqualMillisNanosEvaluator(source, lhs.get(context), rhs.get(context), context); + } + + @Override + public String toString() { + return "GreaterThanOrEqualMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualNanosMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualNanosMillisEvaluator.java new file mode 100644 index 0000000000000..d4386a64aaf8a --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualNanosMillisEvaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link GreaterThanOrEqual}. + * This class is generated. Do not edit it. + */ +public final class GreaterThanOrEqualNanosMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public GreaterThanOrEqualNanosMillisEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + LongVector rhsVector = rhsBlock.asVector(); + if (rhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock rhsBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rhsBlock.getValueCount(p) != 1) { + if (rhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBoolean(GreaterThanOrEqual.processNanosMillis(lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsBlock.getLong(rhsBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, LongVector lhsVector, LongVector rhsVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBoolean(p, GreaterThanOrEqual.processNanosMillis(lhsVector.getLong(p), rhsVector.getLong(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "GreaterThanOrEqualNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, rhs); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory lhs; + + private final EvalOperator.ExpressionEvaluator.Factory rhs; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, + EvalOperator.ExpressionEvaluator.Factory rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public GreaterThanOrEqualNanosMillisEvaluator get(DriverContext context) { + return new GreaterThanOrEqualNanosMillisEvaluator(source, lhs.get(context), rhs.get(context), context); + } + + @Override + public String toString() { + return "GreaterThanOrEqualNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanMillisNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanMillisNanosEvaluator.java new file mode 100644 index 0000000000000..21d7d50af5b1e --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanMillisNanosEvaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link LessThan}. + * This class is generated. Do not edit it. + */ +public final class LessThanMillisNanosEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public LessThanMillisNanosEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + LongVector rhsVector = rhsBlock.asVector(); + if (rhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock rhsBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rhsBlock.getValueCount(p) != 1) { + if (rhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBoolean(LessThan.processMillisNanos(lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsBlock.getLong(rhsBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, LongVector lhsVector, LongVector rhsVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBoolean(p, LessThan.processMillisNanos(lhsVector.getLong(p), rhsVector.getLong(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "LessThanMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, rhs); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory lhs; + + private final EvalOperator.ExpressionEvaluator.Factory rhs; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, + EvalOperator.ExpressionEvaluator.Factory rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public LessThanMillisNanosEvaluator get(DriverContext context) { + return new LessThanMillisNanosEvaluator(source, lhs.get(context), rhs.get(context), context); + } + + @Override + public String toString() { + return "LessThanMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanNanosMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanNanosMillisEvaluator.java new file mode 100644 index 0000000000000..48593f9d537f3 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanNanosMillisEvaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link LessThan}. + * This class is generated. Do not edit it. + */ +public final class LessThanNanosMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public LessThanNanosMillisEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + LongVector rhsVector = rhsBlock.asVector(); + if (rhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock rhsBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rhsBlock.getValueCount(p) != 1) { + if (rhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBoolean(LessThan.processNanosMillis(lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsBlock.getLong(rhsBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, LongVector lhsVector, LongVector rhsVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBoolean(p, LessThan.processNanosMillis(lhsVector.getLong(p), rhsVector.getLong(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "LessThanNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, rhs); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory lhs; + + private final EvalOperator.ExpressionEvaluator.Factory rhs; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, + EvalOperator.ExpressionEvaluator.Factory rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public LessThanNanosMillisEvaluator get(DriverContext context) { + return new LessThanNanosMillisEvaluator(source, lhs.get(context), rhs.get(context), context); + } + + @Override + public String toString() { + return "LessThanNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualMillisNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualMillisNanosEvaluator.java new file mode 100644 index 0000000000000..06973e71e834a --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualMillisNanosEvaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link LessThanOrEqual}. + * This class is generated. Do not edit it. + */ +public final class LessThanOrEqualMillisNanosEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public LessThanOrEqualMillisNanosEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + LongVector rhsVector = rhsBlock.asVector(); + if (rhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock rhsBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rhsBlock.getValueCount(p) != 1) { + if (rhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBoolean(LessThanOrEqual.processMillisNanos(lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsBlock.getLong(rhsBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, LongVector lhsVector, LongVector rhsVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBoolean(p, LessThanOrEqual.processMillisNanos(lhsVector.getLong(p), rhsVector.getLong(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "LessThanOrEqualMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, rhs); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory lhs; + + private final EvalOperator.ExpressionEvaluator.Factory rhs; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, + EvalOperator.ExpressionEvaluator.Factory rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public LessThanOrEqualMillisNanosEvaluator get(DriverContext context) { + return new LessThanOrEqualMillisNanosEvaluator(source, lhs.get(context), rhs.get(context), context); + } + + @Override + public String toString() { + return "LessThanOrEqualMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualNanosMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualNanosMillisEvaluator.java new file mode 100644 index 0000000000000..4763629873d02 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualNanosMillisEvaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link LessThanOrEqual}. + * This class is generated. Do not edit it. + */ +public final class LessThanOrEqualNanosMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public LessThanOrEqualNanosMillisEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + LongVector rhsVector = rhsBlock.asVector(); + if (rhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock rhsBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rhsBlock.getValueCount(p) != 1) { + if (rhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBoolean(LessThanOrEqual.processNanosMillis(lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsBlock.getLong(rhsBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, LongVector lhsVector, LongVector rhsVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBoolean(p, LessThanOrEqual.processNanosMillis(lhsVector.getLong(p), rhsVector.getLong(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "LessThanOrEqualNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, rhs); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory lhs; + + private final EvalOperator.ExpressionEvaluator.Factory rhs; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, + EvalOperator.ExpressionEvaluator.Factory rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public LessThanOrEqualNanosMillisEvaluator get(DriverContext context) { + return new LessThanOrEqualNanosMillisEvaluator(source, lhs.get(context), rhs.get(context), context); + } + + @Override + public String toString() { + return "LessThanOrEqualNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsMillisNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsMillisNanosEvaluator.java new file mode 100644 index 0000000000000..9bede03737a5f --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsMillisNanosEvaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link NotEquals}. + * This class is generated. Do not edit it. + */ +public final class NotEqualsMillisNanosEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public NotEqualsMillisNanosEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + LongVector rhsVector = rhsBlock.asVector(); + if (rhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock rhsBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rhsBlock.getValueCount(p) != 1) { + if (rhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBoolean(NotEquals.processMillisNanos(lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsBlock.getLong(rhsBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, LongVector lhsVector, LongVector rhsVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBoolean(p, NotEquals.processMillisNanos(lhsVector.getLong(p), rhsVector.getLong(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "NotEqualsMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, rhs); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory lhs; + + private final EvalOperator.ExpressionEvaluator.Factory rhs; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, + EvalOperator.ExpressionEvaluator.Factory rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public NotEqualsMillisNanosEvaluator get(DriverContext context) { + return new NotEqualsMillisNanosEvaluator(source, lhs.get(context), rhs.get(context), context); + } + + @Override + public String toString() { + return "NotEqualsMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsNanosMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsNanosMillisEvaluator.java new file mode 100644 index 0000000000000..e8e28eec7ee27 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsNanosMillisEvaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link NotEquals}. + * This class is generated. Do not edit it. + */ +public final class NotEqualsNanosMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public NotEqualsNanosMillisEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + LongVector rhsVector = rhsBlock.asVector(); + if (rhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlock); + } + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); + } + } + } + + public BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock rhsBlock) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rhsBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rhsBlock.getValueCount(p) != 1) { + if (rhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBoolean(NotEquals.processNanosMillis(lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsBlock.getLong(rhsBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public BooleanVector eval(int positionCount, LongVector lhsVector, LongVector rhsVector) { + try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBoolean(p, NotEquals.processNanosMillis(lhsVector.getLong(p), rhsVector.getLong(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "NotEqualsNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, rhs); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory lhs; + + private final EvalOperator.ExpressionEvaluator.Factory rhs; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, + EvalOperator.ExpressionEvaluator.Factory rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public NotEqualsNanosMillisEvaluator get(DriverContext context) { + return new NotEqualsNanosMillisEvaluator(source, lhs.get(context), rhs.get(context), context); + } + + @Override + public String toString() { + return "NotEqualsNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + rhs + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index ee3f5be185b4f..7c3f2a45df6a0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -345,6 +345,11 @@ public enum Cap { */ DATE_NANOS_BINARY_COMPARISON(), + /** + * Support for mixed comparisons between nanosecond and millisecond dates + */ + DATE_NANOS_COMPARE_TO_MILLIS(), + /** * Support Least and Greatest functions on Date Nanos type */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index d6f0ff766eb40..ecfe1aa7f9169 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Enrich; @@ -596,7 +597,11 @@ private void gatherMetrics(LogicalPlan plan, BitSet b) { } /** - * Limit QL's comparisons to types we support. + * Limit QL's comparisons to types we support. This should agree with + * {@link EsqlBinaryComparison}'s checkCompatibility method + * + * @return null if the given binary comparison has valid input types, + * otherwise a failure message suitable to return to the user. */ public static Failure validateBinaryComparison(BinaryComparison bc) { if (bc.left().dataType().isNumeric()) { @@ -641,6 +646,12 @@ public static Failure validateBinaryComparison(BinaryComparison bc) { if (DataType.isString(bc.left().dataType()) && DataType.isString(bc.right().dataType())) { return null; } + + // Allow mixed millisecond and nanosecond binary comparisons + if (bc.left().dataType().isDate() && bc.right().dataType().isDate()) { + return null; + } + if (bc.left().dataType() != bc.right().dataType()) { return fail( bc, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java index 6bb249385affe..464553977d3cc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java @@ -8,6 +8,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -95,11 +96,28 @@ public Equals( description = "An expression." ) Expression right ) { - super(source, left, right, BinaryComparisonOperation.EQ, evaluatorMap); + super( + source, + left, + right, + BinaryComparisonOperation.EQ, + evaluatorMap, + EqualsNanosMillisEvaluator.Factory::new, + EqualsMillisNanosEvaluator.Factory::new + ); } public Equals(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.EQ, zoneId, evaluatorMap); + super( + source, + left, + right, + BinaryComparisonOperation.EQ, + zoneId, + evaluatorMap, + EqualsNanosMillisEvaluator.Factory::new, + EqualsMillisNanosEvaluator.Factory::new + ); } @Override @@ -142,6 +160,16 @@ static boolean processLongs(long lhs, long rhs) { return lhs == rhs; } + @Evaluator(extraName = "MillisNanos") + static boolean processMillisNanos(long lhs, long rhs) { + return DateUtils.compareNanosToMillis(rhs, lhs) == 0; + } + + @Evaluator(extraName = "NanosMillis") + static boolean processNanosMillis(long lhs, long rhs) { + return DateUtils.compareNanosToMillis(lhs, rhs) == 0; + } + @Evaluator(extraName = "Doubles") static boolean processDoubles(double lhs, double rhs) { return lhs == rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index cbbf87fb6c4cb..217c6528c9fd6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -35,6 +35,8 @@ public abstract class EsqlBinaryComparison extends BinaryComparison implements E private final Map evaluatorMap; private final BinaryComparisonOperation functionType; + private final EsqlArithmeticOperation.BinaryEvaluator nanosToMillisEvaluator; + private final EsqlArithmeticOperation.BinaryEvaluator millisToNanosEvaluator; @FunctionalInterface public interface BinaryOperatorConstructor { @@ -118,9 +120,11 @@ protected EsqlBinaryComparison( Expression left, Expression right, BinaryComparisonOperation operation, - Map evaluatorMap + Map evaluatorMap, + EsqlArithmeticOperation.BinaryEvaluator nanosToMillisEvaluator, + EsqlArithmeticOperation.BinaryEvaluator millisToNanosEvaluator ) { - this(source, left, right, operation, null, evaluatorMap); + this(source, left, right, operation, null, evaluatorMap, nanosToMillisEvaluator, millisToNanosEvaluator); } protected EsqlBinaryComparison( @@ -130,11 +134,15 @@ protected EsqlBinaryComparison( BinaryComparisonOperation operation, // TODO: We are definitely not doing the right thing with this zoneId ZoneId zoneId, - Map evaluatorMap + Map evaluatorMap, + EsqlArithmeticOperation.BinaryEvaluator nanosToMillisEvaluator, + EsqlArithmeticOperation.BinaryEvaluator millisToNanosEvaluator ) { super(source, left, right, operation.shim, zoneId); this.evaluatorMap = evaluatorMap; this.functionType = operation; + this.nanosToMillisEvaluator = nanosToMillisEvaluator; + this.millisToNanosEvaluator = millisToNanosEvaluator; } public static EsqlBinaryComparison readFrom(StreamInput in) throws IOException { @@ -163,11 +171,24 @@ public BinaryComparisonOperation getFunctionType() { @Override public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { - // Our type is always boolean, so figure out the evaluator type from the inputs - DataType commonType = commonType(left().dataType(), right().dataType()); EvalOperator.ExpressionEvaluator.Factory lhs; EvalOperator.ExpressionEvaluator.Factory rhs; + // Special cases for mixed nanosecond and millisecond comparisions + if (left().dataType() == DataType.DATE_NANOS && right().dataType() == DataType.DATETIME) { + lhs = toEvaluator.apply(left()); + rhs = toEvaluator.apply(right()); + return nanosToMillisEvaluator.apply(source(), lhs, rhs); + } + + if (left().dataType() == DataType.DATETIME && right().dataType() == DataType.DATE_NANOS) { + lhs = toEvaluator.apply(left()); + rhs = toEvaluator.apply(right()); + return millisToNanosEvaluator.apply(source(), lhs, rhs); + } + + // Our type is always boolean, so figure out the evaluator type from the inputs + DataType commonType = commonType(left().dataType(), right().dataType()); if (commonType.isNumeric()) { lhs = Cast.cast(source(), left().dataType(), commonType, toEvaluator.apply(left())); rhs = Cast.cast(source(), right().dataType(), commonType, toEvaluator.apply(right())); @@ -209,7 +230,9 @@ protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrd } /** - * Check if the two input types are compatible for this operation + * Check if the two input types are compatible for this operation. + * NOTE: this method should be consistent with + * {@link org.elasticsearch.xpack.esql.analysis.Verifier#validateBinaryComparison(BinaryComparison)} * * @return TypeResolution.TYPE_RESOLVED iff the types are compatible. Otherwise, an appropriate type resolution error. */ @@ -225,6 +248,7 @@ protected TypeResolution checkCompatibility() { if ((leftType.isNumeric() && rightType.isNumeric()) || (DataType.isString(leftType) && DataType.isString(rightType)) + || (leftType.isDate() && rightType.isDate()) // Millis and Nanos || leftType.equals(rightType) || DataType.isNull(leftType) || DataType.isNull(rightType)) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java index 3a46070389368..6087240387f01 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java @@ -8,6 +8,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -62,11 +63,28 @@ public GreaterThan( description = "An expression." ) Expression right ) { - super(source, left, right, BinaryComparisonOperation.GT, evaluatorMap); + super( + source, + left, + right, + BinaryComparisonOperation.GT, + evaluatorMap, + GreaterThanNanosMillisEvaluator.Factory::new, + GreaterThanMillisNanosEvaluator.Factory::new + ); } public GreaterThan(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.GT, zoneId, evaluatorMap); + super( + source, + left, + right, + BinaryComparisonOperation.GT, + zoneId, + evaluatorMap, + GreaterThanNanosMillisEvaluator.Factory::new, + GreaterThanMillisNanosEvaluator.Factory::new + ); } @Override @@ -109,6 +127,17 @@ static boolean processLongs(long lhs, long rhs) { return lhs > rhs; } + @Evaluator(extraName = "MillisNanos") + static boolean processMillisNanos(long lhs, long rhs) { + // Note, parameters are reversed, so we need to invert the check. + return DateUtils.compareNanosToMillis(rhs, lhs) < 0; + } + + @Evaluator(extraName = "NanosMillis") + static boolean processNanosMillis(long lhs, long rhs) { + return DateUtils.compareNanosToMillis(lhs, rhs) > 0; + } + @Evaluator(extraName = "Doubles") static boolean processDoubles(double lhs, double rhs) { return lhs > rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java index 841fe5294c660..7ec1e5590bef6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java @@ -8,6 +8,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -62,11 +63,28 @@ public GreaterThanOrEqual( description = "An expression." ) Expression right ) { - super(source, left, right, BinaryComparisonOperation.GTE, evaluatorMap); + super( + source, + left, + right, + BinaryComparisonOperation.GTE, + evaluatorMap, + GreaterThanOrEqualNanosMillisEvaluator.Factory::new, + GreaterThanOrEqualMillisNanosEvaluator.Factory::new + ); } public GreaterThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.GTE, zoneId, evaluatorMap); + super( + source, + left, + right, + BinaryComparisonOperation.GTE, + zoneId, + evaluatorMap, + GreaterThanOrEqualNanosMillisEvaluator.Factory::new, + GreaterThanOrEqualMillisNanosEvaluator.Factory::new + ); } @Override @@ -109,6 +127,17 @@ static boolean processLongs(long lhs, long rhs) { return lhs >= rhs; } + @Evaluator(extraName = "MillisNanos") + static boolean processMillisNanos(long lhs, long rhs) { + // Note, parameters are reversed, so we need to invert the check. + return DateUtils.compareNanosToMillis(rhs, lhs) <= 0; + } + + @Evaluator(extraName = "NanosMillis") + static boolean processNanosMillis(long lhs, long rhs) { + return DateUtils.compareNanosToMillis(lhs, rhs) >= 0; + } + @Evaluator(extraName = "Doubles") static boolean processDoubles(double lhs, double rhs) { return lhs >= rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java index 3ae7bd93092ef..5f130c054cd6f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java @@ -8,6 +8,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -66,7 +67,16 @@ public LessThan( } public LessThan(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.LT, zoneId, evaluatorMap); + super( + source, + left, + right, + BinaryComparisonOperation.LT, + zoneId, + evaluatorMap, + LessThanNanosMillisEvaluator.Factory::new, + LessThanMillisNanosEvaluator.Factory::new + ); } @Override @@ -109,6 +119,17 @@ static boolean processLongs(long lhs, long rhs) { return lhs < rhs; } + @Evaluator(extraName = "MillisNanos") + static boolean processMillisNanos(long lhs, long rhs) { + // Note, parameters are reversed, so we need to invert the check. + return DateUtils.compareNanosToMillis(rhs, lhs) > 0; + } + + @Evaluator(extraName = "NanosMillis") + static boolean processNanosMillis(long lhs, long rhs) { + return DateUtils.compareNanosToMillis(lhs, rhs) < 0; + } + @Evaluator(extraName = "Doubles") static boolean processDoubles(double lhs, double rhs) { return lhs < rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java index e084eee1e8c20..0904c408bfab5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java @@ -8,6 +8,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -66,7 +67,16 @@ public LessThanOrEqual( } public LessThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.LTE, zoneId, evaluatorMap); + super( + source, + left, + right, + BinaryComparisonOperation.LTE, + zoneId, + evaluatorMap, + LessThanOrEqualNanosMillisEvaluator.Factory::new, + LessThanOrEqualMillisNanosEvaluator.Factory::new + ); } @Override @@ -109,6 +119,17 @@ static boolean processLongs(long lhs, long rhs) { return lhs <= rhs; } + @Evaluator(extraName = "MillisNanos") + static boolean processMillisNanos(long lhs, long rhs) { + // Note, parameters are reversed, so we need to invert the check. + return DateUtils.compareNanosToMillis(rhs, lhs) >= 0; + } + + @Evaluator(extraName = "NanosMillis") + static boolean processNanosMillis(long lhs, long rhs) { + return DateUtils.compareNanosToMillis(lhs, rhs) <= 0; + } + @Evaluator(extraName = "Doubles") static boolean processDoubles(double lhs, double rhs) { return lhs <= rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java index 9e961c04153d6..d4f86e9a878a9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java @@ -8,6 +8,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -95,11 +96,28 @@ public NotEquals( description = "An expression." ) Expression right ) { - super(source, left, right, BinaryComparisonOperation.NEQ, evaluatorMap); + super( + source, + left, + right, + BinaryComparisonOperation.NEQ, + evaluatorMap, + NotEqualsNanosMillisEvaluator.Factory::new, + NotEqualsMillisNanosEvaluator.Factory::new + ); } public NotEquals(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonOperation.NEQ, zoneId, evaluatorMap); + super( + source, + left, + right, + BinaryComparisonOperation.NEQ, + zoneId, + evaluatorMap, + NotEqualsNanosMillisEvaluator.Factory::new, + NotEqualsMillisNanosEvaluator.Factory::new + ); } @Override @@ -117,6 +135,16 @@ static boolean processLongs(long lhs, long rhs) { return lhs != rhs; } + @Evaluator(extraName = "MillisNanos") + static boolean processMillisNanos(long lhs, long rhs) { + return DateUtils.compareNanosToMillis(rhs, lhs) != 0; + } + + @Evaluator(extraName = "NanosMillis") + static boolean processNanosMillis(long lhs, long rhs) { + return DateUtils.compareNanosToMillis(lhs, rhs) != 0; + } + @Evaluator(extraName = "Doubles") static boolean processDoubles(double lhs, double rhs) { return lhs != rhs; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 377027b70fb54..2004fa3a1cdb0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -1791,9 +1791,9 @@ public TypedData withData(Object data) { @Override public String toString() { if (type == DataType.UNSIGNED_LONG && data instanceof Long longData) { - return type.toString() + "(" + NumericUtils.unsignedLongAsBigInteger(longData).toString() + ")"; + return type + "(" + NumericUtils.unsignedLongAsBigInteger(longData).toString() + ")"; } - return type.toString() + "(" + (data == null ? "null" : data.toString()) + ")"; + return type.toString() + "(" + (data == null ? "null" : getValue().toString()) + ")"; } /** diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java index 0fb416584b472..6666eb8adab61 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java @@ -144,6 +144,34 @@ public static Iterable parameters() { ) ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsNanosMillisEvaluator", + "lhs", + "rhs", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.dateNanosCases(), + TestCaseSupplier.dateCases(), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsMillisNanosEvaluator", + "lhs", + "rhs", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.dateCases(), + TestCaseSupplier.dateNanosCases(), + List.of(), + false + ) + ); + suppliers.addAll( TestCaseSupplier.stringCases( Object::equals, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index 395a574028f6a..0fbd49abd885b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -121,6 +121,34 @@ public static Iterable parameters() { throw new UnsupportedOperationException("Got some weird types"); }, DataType.BOOLEAN, TestCaseSupplier.dateNanosCases(), TestCaseSupplier.dateNanosCases(), List.of(), false)); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "GreaterThanOrEqualNanosMillisEvaluator", + "lhs", + "rhs", + (lhs, rhs) -> (((Instant) lhs).isAfter((Instant) rhs) || lhs.equals(rhs)), + DataType.BOOLEAN, + TestCaseSupplier.dateNanosCases(), + TestCaseSupplier.dateCases(), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "GreaterThanOrEqualMillisNanosEvaluator", + "lhs", + "rhs", + (lhs, rhs) -> (((Instant) lhs).isAfter((Instant) rhs) || lhs.equals(rhs)), + DataType.BOOLEAN, + TestCaseSupplier.dateCases(), + TestCaseSupplier.dateNanosCases(), + List.of(), + false + ) + ); + suppliers.addAll( TestCaseSupplier.stringCases( (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) >= 0, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index b56ecd7392ba6..ccc66df60fb3f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -135,6 +135,34 @@ public static Iterable parameters() { ) ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "GreaterThanNanosMillisEvaluator", + "lhs", + "rhs", + (l, r) -> ((Instant) l).isAfter((Instant) r), + DataType.BOOLEAN, + TestCaseSupplier.dateNanosCases(), + TestCaseSupplier.dateCases(), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "GreaterThanMillisNanosEvaluator", + "lhs", + "rhs", + (l, r) -> ((Instant) l).isAfter((Instant) r), + DataType.BOOLEAN, + TestCaseSupplier.dateCases(), + TestCaseSupplier.dateNanosCases(), + List.of(), + false + ) + ); + suppliers.addAll( TestCaseSupplier.stringCases( (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) > 0, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index 60062f071c183..1e91a65e04c0e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -121,6 +121,34 @@ public static Iterable parameters() { throw new UnsupportedOperationException("Got some weird types"); }, DataType.BOOLEAN, TestCaseSupplier.dateNanosCases(), TestCaseSupplier.dateNanosCases(), List.of(), false)); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "LessThanOrEqualNanosMillisEvaluator", + "lhs", + "rhs", + (l, r) -> (((Instant) l).isBefore((Instant) r) || l.equals(r)), + DataType.BOOLEAN, + TestCaseSupplier.dateNanosCases(), + TestCaseSupplier.dateCases(), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "LessThanOrEqualMillisNanosEvaluator", + "lhs", + "rhs", + (l, r) -> (((Instant) l).isBefore((Instant) r) || l.equals(r)), + DataType.BOOLEAN, + TestCaseSupplier.dateCases(), + TestCaseSupplier.dateNanosCases(), + List.of(), + false + ) + ); + suppliers.addAll( TestCaseSupplier.stringCases( (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) <= 0, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index 30812cf8e538d..69dc59bac6456 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -135,6 +135,34 @@ public static Iterable parameters() { ) ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "LessThanNanosMillisEvaluator", + "lhs", + "rhs", + (l, r) -> ((Instant) l).isBefore((Instant) r), + DataType.BOOLEAN, + TestCaseSupplier.dateNanosCases(), + TestCaseSupplier.dateCases(), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "LessThanMillisNanosEvaluator", + "lhs", + "rhs", + (l, r) -> ((Instant) l).isBefore((Instant) r), + DataType.BOOLEAN, + TestCaseSupplier.dateCases(), + TestCaseSupplier.dateNanosCases(), + List.of(), + false + ) + ); + suppliers.addAll( TestCaseSupplier.stringCases( (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) < 0, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java index 53676a43b16a0..7b57b97dfe28e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java @@ -128,7 +128,7 @@ public static Iterable parameters() { false ) ); - // Datetime + // Datenanos suppliers.addAll( TestCaseSupplier.forBinaryNotCasting( "NotEqualsLongsEvaluator", @@ -142,6 +142,36 @@ public static Iterable parameters() { false ) ); + + // nanoseconds to milliseconds. NB: these have different evaluator names depending on the direction + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsNanosMillisEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataType.BOOLEAN, + TestCaseSupplier.dateNanosCases(), + TestCaseSupplier.dateCases(), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsMillisNanosEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataType.BOOLEAN, + TestCaseSupplier.dateCases(), + TestCaseSupplier.dateNanosCases(), + List.of(), + false + ) + ); + suppliers.addAll( TestCaseSupplier.stringCases( (l, r) -> false == l.equals(r), From ca09e728711bc1bb39b7c85ac313d255a797a53c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 7 Dec 2024 01:48:00 +1100 Subject: [PATCH 082/204] Mute org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT testEsqlRcs2UnavailableRemoteScenarios #117419 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 6d48052fe25e1..fb2fea908ef9e 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -257,6 +257,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT method: test {lookup-join.LookupMessageFromIndexKeepReordered ASYNC} issue: https://github.com/elastic/elasticsearch/issues/118151 +- class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT + method: testEsqlRcs2UnavailableRemoteScenarios + issue: https://github.com/elastic/elasticsearch/issues/117419 # Examples: # From e55f07b85037e1c59155b5d08495a32f9b1b495d Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Fri, 6 Dec 2024 10:05:17 -0500 Subject: [PATCH 083/204] Remove long deprecated ignore_case from synonym filter (#115985) ignore_case has been deprecated in synonym filter since ES 6. --- .../analysis/common/SynonymTokenFilterFactory.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java index 9e31fdde4330b..9dc3478994f1f 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java @@ -13,7 +13,6 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.synonym.SynonymFilter; import org.apache.lucene.analysis.synonym.SynonymMap; -import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; @@ -152,15 +151,6 @@ public static SynonymsSource fromSettings(Settings settings) { super(name, settings); this.settings = settings; - if (settings.get("ignore_case") != null) { - DEPRECATION_LOGGER.warn( - DeprecationCategory.ANALYSIS, - "synonym_ignore_case_option", - "The ignore_case option on the synonym_graph filter is deprecated. " - + "Instead, insert a lowercase filter in the filter chain before the synonym_graph filter." - ); - } - this.synonymsSource = SynonymsSource.fromSettings(settings); this.expand = settings.getAsBoolean("expand", true); this.format = settings.get("format", ""); From bb60b1d0f4b35a51293485e38101396e29248b83 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 6 Dec 2024 09:25:17 -0600 Subject: [PATCH 084/204] muting KeystoreManagementTests.test31WrongKeystorePasswordFromFile --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index fb2fea908ef9e..9353534a9d830 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -248,6 +248,9 @@ tests: - class: org.elasticsearch.packaging.test.KeystoreManagementTests method: test30KeystorePasswordFromFile issue: https://github.com/elastic/elasticsearch/issues/118123 +- class: org.elasticsearch.packaging.test.KeystoreManagementTests + method: test31WrongKeystorePasswordFromFile + issue: https://github.com/elastic/elasticsearch/issues/118123 - class: org.elasticsearch.packaging.test.ArchiveTests method: test41AutoconfigurationNotTriggeredWhenNodeCannotContainData issue: https://github.com/elastic/elasticsearch/issues/118110 From 3ae2330630e5f5d0fa62e4c41288a6dbae47678f Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 6 Dec 2024 07:33:06 -0800 Subject: [PATCH 085/204] Ignore cancellation exceptions (#117657) Today, when an ES|QL task encounters an exception, we trigger a cancellation on the root task, causing child tasks to fail due to cancellation. We chose not to include cancellation exceptions in the output, as they are unhelpful and add noise during problem analysis. However, these exceptions are still slipping through via RefCountingListener. This change addresses the issue by introducing ESQLRefCountingListener, ensuring that no cancellation exceptions are returned. --- docs/changelog/117657.yaml | 5 ++ .../compute/EsqlRefCountingListener.java | 47 ++++++++++++++++++ .../compute/operator/FailureCollector.java | 48 +++++++++---------- .../exchange/ExchangeSourceHandler.java | 17 +++---- .../operator/FailureCollectorTests.java | 9 ++++ .../xpack/esql/EsqlTestUtils.java | 18 +++++++ .../xpack/esql/action/EnrichIT.java | 1 + .../esql/action/EsqlActionBreakerIT.java | 2 + .../xpack/esql/action/EsqlActionTaskIT.java | 12 ++++- .../xpack/esql/action/EsqlDisruptionIT.java | 2 + .../xpack/esql/plugin/ComputeListener.java | 10 ++-- .../xpack/esql/plugin/ComputeService.java | 6 +-- 12 files changed, 135 insertions(+), 42 deletions(-) create mode 100644 docs/changelog/117657.yaml create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/EsqlRefCountingListener.java diff --git a/docs/changelog/117657.yaml b/docs/changelog/117657.yaml new file mode 100644 index 0000000000000..0a72e9dabe9e8 --- /dev/null +++ b/docs/changelog/117657.yaml @@ -0,0 +1,5 @@ +pr: 117657 +summary: Ignore cancellation exceptions +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/EsqlRefCountingListener.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/EsqlRefCountingListener.java new file mode 100644 index 0000000000000..69df0fb8ceff1 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/EsqlRefCountingListener.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.RefCountingRunnable; +import org.elasticsearch.compute.operator.FailureCollector; +import org.elasticsearch.core.Releasable; + +/** + * Similar to {@link org.elasticsearch.action.support.RefCountingListener}, + * but prefers non-task-cancelled exceptions over task-cancelled ones as they are more useful for diagnosing issues. + * @see FailureCollector + */ +public final class EsqlRefCountingListener implements Releasable { + private final FailureCollector failureCollector; + private final RefCountingRunnable refs; + + public EsqlRefCountingListener(ActionListener delegate) { + this.failureCollector = new FailureCollector(); + this.refs = new RefCountingRunnable(() -> { + Exception error = failureCollector.getFailure(); + if (error != null) { + delegate.onFailure(error); + } else { + delegate.onResponse(null); + } + }); + } + + public ActionListener acquire() { + return refs.acquireListener().delegateResponse((l, e) -> { + failureCollector.unwrapAndCollect(e); + l.onFailure(e); + }); + } + + @Override + public void close() { + refs.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FailureCollector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FailureCollector.java index 943ba4dc1f4fa..337075edbdcf6 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FailureCollector.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FailureCollector.java @@ -13,9 +13,8 @@ import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.transport.TransportException; -import java.util.List; import java.util.Queue; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.Semaphore; /** * {@code FailureCollector} is responsible for collecting exceptions that occur in the compute engine. @@ -26,12 +25,11 @@ */ public final class FailureCollector { private final Queue cancelledExceptions = ConcurrentCollections.newQueue(); - private final AtomicInteger cancelledExceptionsCount = new AtomicInteger(); + private final Semaphore cancelledExceptionsPermits; private final Queue nonCancelledExceptions = ConcurrentCollections.newQueue(); - private final AtomicInteger nonCancelledExceptionsCount = new AtomicInteger(); + private final Semaphore nonCancelledExceptionsPermits; - private final int maxExceptions; private volatile boolean hasFailure = false; private Exception finalFailure = null; @@ -43,7 +41,8 @@ public FailureCollector(int maxExceptions) { if (maxExceptions <= 0) { throw new IllegalArgumentException("maxExceptions must be at least one"); } - this.maxExceptions = maxExceptions; + this.cancelledExceptionsPermits = new Semaphore(maxExceptions); + this.nonCancelledExceptionsPermits = new Semaphore(maxExceptions); } private static Exception unwrapTransportException(TransportException te) { @@ -60,13 +59,12 @@ private static Exception unwrapTransportException(TransportException te) { public void unwrapAndCollect(Exception e) { e = e instanceof TransportException te ? unwrapTransportException(te) : e; if (ExceptionsHelper.unwrap(e, TaskCancelledException.class) != null) { - if (cancelledExceptionsCount.incrementAndGet() <= maxExceptions) { + if (nonCancelledExceptions.isEmpty() && cancelledExceptionsPermits.tryAcquire()) { cancelledExceptions.add(e); } - } else { - if (nonCancelledExceptionsCount.incrementAndGet() <= maxExceptions) { - nonCancelledExceptions.add(e); - } + } else if (nonCancelledExceptionsPermits.tryAcquire()) { + nonCancelledExceptions.add(e); + cancelledExceptions.clear(); } hasFailure = true; } @@ -99,20 +97,22 @@ public Exception getFailure() { private Exception buildFailure() { assert hasFailure; assert Thread.holdsLock(this); - int total = 0; Exception first = null; - for (var exceptions : List.of(nonCancelledExceptions, cancelledExceptions)) { - for (Exception e : exceptions) { - if (first == null) { - first = e; - total++; - } else if (first != e) { - first.addSuppressed(e); - total++; - } - if (total >= maxExceptions) { - return first; - } + for (Exception e : nonCancelledExceptions) { + if (first == null) { + first = e; + } else if (first != e) { + first.addSuppressed(e); + } + } + if (first != null) { + return first; + } + for (Exception e : cancelledExceptions) { + if (first == null) { + first = e; + } else if (first != e) { + first.addSuppressed(e); } } assert first != null; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java index 375016a5d51d5..b53ddea3da587 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java @@ -9,9 +9,10 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.RefCountingListener; +import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.compute.EsqlRefCountingListener; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.FailureCollector; import org.elasticsearch.compute.operator.IsBlockedResult; @@ -54,20 +55,20 @@ public ExchangeSourceHandler(int maxBufferSize, Executor fetchExecutor, ActionLi this.outstandingSinks = new PendingInstances(() -> buffer.finish(false)); this.outstandingSources = new PendingInstances(() -> buffer.finish(true)); buffer.addCompletionListener(ActionListener.running(() -> { - final ActionListener listener = ActionListener.assertAtLeastOnce(completionListener).delegateFailure((l, unused) -> { + final ActionListener listener = ActionListener.assertAtLeastOnce(completionListener); + try (RefCountingRunnable refs = new RefCountingRunnable(() -> { final Exception e = failure.getFailure(); if (e != null) { - l.onFailure(e); + listener.onFailure(e); } else { - l.onResponse(null); + listener.onResponse(null); } - }); - try (RefCountingListener refs = new RefCountingListener(listener)) { + })) { for (PendingInstances pending : List.of(outstandingSinks, outstandingSources)) { // Create an outstanding instance and then finish to complete the completionListener // if we haven't registered any instances of exchange sinks or exchange sources before. pending.trackNewInstance(); - pending.completion.addListener(refs.acquire()); + pending.completion.addListener(refs.acquireListener()); pending.finishInstance(); } } @@ -269,7 +270,7 @@ public void onFailure(Exception e) { @Override protected void doRun() { - try (RefCountingListener refs = new RefCountingListener(sinkListener)) { + try (EsqlRefCountingListener refs = new EsqlRefCountingListener(sinkListener)) { for (int i = 0; i < instances; i++) { var fetcher = new RemoteSinkFetcher(remoteSink, failFast, refs.acquire()); fetcher.fetchPage(); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FailureCollectorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FailureCollectorTests.java index 637cbe8892b3e..5fec82b32ddac 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FailureCollectorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FailureCollectorTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.operator; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -86,6 +87,14 @@ public void testCollect() throws Exception { assertNotNull(failure); assertThat(failure, Matchers.in(nonCancelledExceptions)); assertThat(failure.getSuppressed().length, lessThan(maxExceptions)); + assertTrue( + "cancellation exceptions must be ignored", + ExceptionsHelper.unwrapCausesAndSuppressed(failure, t -> t instanceof TaskCancelledException).isEmpty() + ); + assertTrue( + "remote transport exception must be unwrapped", + ExceptionsHelper.unwrapCausesAndSuppressed(failure, t -> t instanceof TransportException).isEmpty() + ); } public void testEmpty() { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index ec9af33dd6690..5535e801b1b0c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -10,6 +10,7 @@ import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.Strings; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.NoopCircuitBreaker; @@ -30,7 +31,9 @@ import org.elasticsearch.geo.ShapeTestUtils; import org.elasticsearch.index.IndexMode; import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; @@ -129,6 +132,8 @@ import static org.elasticsearch.xpack.esql.parser.ParserUtils.ParamClassification.PATTERN; import static org.elasticsearch.xpack.esql.parser.ParserUtils.ParamClassification.VALUE; import static org.hamcrest.Matchers.instanceOf; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; public final class EsqlTestUtils { @@ -784,4 +789,17 @@ public static QueryParam paramAsIdentifier(String name, Object value) { public static QueryParam paramAsPattern(String name, Object value) { return new QueryParam(name, value, NULL, PATTERN); } + + /** + * Asserts that: + * 1. Cancellation exceptions are ignored when more relevant exceptions exist. + * 2. Transport exceptions are unwrapped, and the actual causes are reported to users. + */ + public static void assertEsqlFailure(Exception e) { + assertNotNull(e); + var cancellationFailure = ExceptionsHelper.unwrapCausesAndSuppressed(e, t -> t instanceof TaskCancelledException).orElse(null); + assertNull("cancellation exceptions must be ignored", cancellationFailure); + ExceptionsHelper.unwrapCausesAndSuppressed(e, t -> t instanceof RemoteTransportException) + .ifPresent(transportFailure -> assertNull("remote transport exception must be unwrapped", transportFailure.getCause())); + } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java index dab99a0f719dd..c4da0bf32ef96 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java @@ -143,6 +143,7 @@ protected EsqlQueryResponse run(EsqlQueryRequest request) { return client.execute(EsqlQueryAction.INSTANCE, request).actionGet(2, TimeUnit.MINUTES); } catch (Exception e) { logger.info("request failed", e); + EsqlTestUtils.assertEsqlFailure(e); ensureBlocksReleased(); } finally { setRequestCircuitBreakerLimit(null); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java index 37833d8aed2d3..ec7ee8b61c2d5 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import java.util.ArrayList; import java.util.Collection; @@ -85,6 +86,7 @@ private EsqlQueryResponse runWithBreaking(EsqlQueryRequest request) throws Circu } catch (Exception e) { logger.info("request failed", e); ensureBlocksReleased(); + EsqlTestUtils.assertEsqlFailure(e); throw e; } finally { setRequestCircuitBreakerLimit(null); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java index 1939f81353c0e..abd4f6b49d7b4 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java @@ -36,6 +36,7 @@ import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.junit.Before; @@ -338,7 +339,15 @@ private void assertCancelled(ActionFuture response) throws Ex */ assertThat( cancelException.getMessage(), - in(List.of("test cancel", "task cancelled", "request cancelled test cancel", "parent task was cancelled [test cancel]")) + in( + List.of( + "test cancel", + "task cancelled", + "request cancelled test cancel", + "parent task was cancelled [test cancel]", + "cancelled on failure" + ) + ) ); assertBusy( () -> assertThat( @@ -434,6 +443,7 @@ protected void doRun() throws Exception { allowedFetching.countDown(); } Exception failure = expectThrows(Exception.class, () -> future.actionGet().close()); + EsqlTestUtils.assertEsqlFailure(failure); assertThat(failure.getMessage(), containsString("failed to fetch pages")); // If we proceed without waiting for pages, we might cancel the main request before starting the data-node request. // As a result, the exchange sinks on data-nodes won't be removed until the inactive_timeout elapses, which is diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlDisruptionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlDisruptionIT.java index e9eada5def0dc..72a60a6b6b928 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlDisruptionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlDisruptionIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.test.disruption.ServiceDisruptionScheme; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportSettings; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import java.util.ArrayList; import java.util.Collection; @@ -111,6 +112,7 @@ private EsqlQueryResponse runQueryWithDisruption(EsqlQueryRequest request) { assertTrue("request must be failed or completed after clearing disruption", future.isDone()); ensureBlocksReleased(); logger.info("--> failed to execute esql query with disruption; retrying...", e); + EsqlTestUtils.assertEsqlFailure(e); return client().execute(EsqlQueryAction.INSTANCE, request).actionGet(2, TimeUnit.MINUTES); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java index 8d041ffbdf0e4..8bd23230fcde7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeListener.java @@ -9,8 +9,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.RefCountingListener; +import org.elasticsearch.compute.EsqlRefCountingListener; import org.elasticsearch.compute.operator.DriverProfile; -import org.elasticsearch.compute.operator.FailureCollector; import org.elasticsearch.compute.operator.ResponseHeadersCollector; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; @@ -39,8 +39,7 @@ final class ComputeListener implements Releasable { private static final Logger LOGGER = LogManager.getLogger(ComputeService.class); - private final RefCountingListener refs; - private final FailureCollector failureCollector = new FailureCollector(); + private final EsqlRefCountingListener refs; private final AtomicBoolean cancelled = new AtomicBoolean(); private final CancellableTask task; private final TransportService transportService; @@ -105,7 +104,7 @@ private ComputeListener( : "clusterAlias and executionInfo must both be null or both non-null"; // listener that executes after all the sub-listeners refs (created via acquireCompute) have completed - this.refs = new RefCountingListener(1, ActionListener.wrap(ignored -> { + this.refs = new EsqlRefCountingListener(delegate.delegateFailure((l, ignored) -> { responseHeaders.finish(); ComputeResponse result; @@ -131,7 +130,7 @@ private ComputeListener( } } delegate.onResponse(result); - }, e -> delegate.onFailure(failureCollector.getFailure()))); + })); } private static void setFinalStatusAndShardCounts(String clusterAlias, EsqlExecutionInfo executionInfo) { @@ -191,7 +190,6 @@ private boolean isCCSListener(String computeClusterAlias) { */ ActionListener acquireAvoid() { return refs.acquire().delegateResponse((l, e) -> { - failureCollector.unwrapAndCollect(e); try { if (cancelled.compareAndSet(false, true)) { LOGGER.debug("cancelling ESQL task {} on failure", task); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index ed037d24139f8..9b59b98a7cdc2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -16,11 +16,11 @@ import org.elasticsearch.action.search.SearchShardsRequest; import org.elasticsearch.action.search.SearchShardsResponse; import org.elasticsearch.action.support.ChannelActionListener; -import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.EsqlRefCountingListener; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.Driver; @@ -375,7 +375,7 @@ private void startComputeOnDataNodes( var lookupListener = ActionListener.releaseAfter(computeListener.acquireAvoid(), exchangeSource.addEmptySink()); // SearchShards API can_match is done in lookupDataNodes lookupDataNodes(parentTask, clusterAlias, requestFilter, concreteIndices, originalIndices, ActionListener.wrap(dataNodeResult -> { - try (RefCountingListener refs = new RefCountingListener(lookupListener)) { + try (EsqlRefCountingListener refs = new EsqlRefCountingListener(lookupListener)) { // update ExecutionInfo with shard counts (total and skipped) executionInfo.swapCluster( clusterAlias, @@ -436,7 +436,7 @@ private void startComputeOnRemoteClusters( ) { var queryPragmas = configuration.pragmas(); var linkExchangeListeners = ActionListener.releaseAfter(computeListener.acquireAvoid(), exchangeSource.addEmptySink()); - try (RefCountingListener refs = new RefCountingListener(linkExchangeListeners)) { + try (EsqlRefCountingListener refs = new EsqlRefCountingListener(linkExchangeListeners)) { for (RemoteCluster cluster : clusters) { final var childSessionId = newChildSession(sessionId); ExchangeService.openExchange( From 4f030efcd50781fe6e624dc08046b834786edb4a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 7 Dec 2024 02:37:17 +1100 Subject: [PATCH 086/204] Mute org.elasticsearch.packaging.test.DebPreservationTests test40RestartOnUpgrade #118170 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 9353534a9d830..8f1030279efc9 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -263,6 +263,9 @@ tests: - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT method: testEsqlRcs2UnavailableRemoteScenarios issue: https://github.com/elastic/elasticsearch/issues/117419 +- class: org.elasticsearch.packaging.test.DebPreservationTests + method: test40RestartOnUpgrade + issue: https://github.com/elastic/elasticsearch/issues/118170 # Examples: # From 8c380079864c3b98fef872b90102bf1705994f96 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 6 Dec 2024 12:48:37 -0500 Subject: [PATCH 087/204] ESQL: Rework `isNull` (#118101) This reworks `Expressions#isNull` so it only matches the `null` literal. This doesn't super change the way ESQL works because we already rewrite things that `fold` into `null` into the `null` literal. It's just that, now, `isNull` won't return `true` for things that *fold* to null - only things that have *already* folded to null. This is important because `fold` can be quite expensive so we're better off keeping the results of it when possible. Which is what the constant folding rules *do*. --- .../xpack/esql/core/expression/Expressions.java | 12 ++++++++++-- .../expression/predicate/operator/comparison/In.java | 6 +++--- .../xpack/esql/optimizer/rules/logical/FoldNull.java | 6 +++--- .../esql/optimizer/rules/logical/PruneFilters.java | 8 ++++---- .../rules/logical/SplitInWithFoldableValue.java | 2 +- .../esql/optimizer/LogicalPlanOptimizerTests.java | 2 +- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java index 8baffbf887e47..4e4338aad3704 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java @@ -132,8 +132,16 @@ public static String name(Expression e) { return e instanceof NamedExpression ne ? ne.name() : e.sourceText(); } - public static boolean isNull(Expression e) { - return e.dataType() == DataType.NULL || (e.foldable() && e.fold() == null); + /** + * Is this {@linkplain Expression} guaranteed to have + * only the {@code null} value. {@linkplain Expression}s that + * {@link Expression#fold()} to {@code null} may + * return {@code false} here, but should eventually be folded + * into a {@link Literal} containing {@code null} which will return + * {@code true} from here. + */ + public static boolean isGuaranteedNull(Expression e) { + return e.dataType() == DataType.NULL || (e instanceof Literal lit && lit.value() == null); } public static List names(Collection e) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java index eda6aadccc86a..f6c23304c189b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java @@ -151,14 +151,14 @@ public Expression replaceChildren(List newChildren) { public boolean foldable() { // QL's In fold()s to null, if value() is null, but isn't foldable() unless all children are // TODO: update this null check in QL too? - return Expressions.isNull(value) + return Expressions.isGuaranteedNull(value) || Expressions.foldable(children()) - || (Expressions.foldable(list) && list.stream().allMatch(Expressions::isNull)); + || (Expressions.foldable(list) && list.stream().allMatch(Expressions::isGuaranteedNull)); } @Override public Object fold() { - if (Expressions.isNull(value) || list.stream().allMatch(Expressions::isNull)) { + if (Expressions.isGuaranteedNull(value) || list.stream().allMatch(Expressions::isGuaranteedNull)) { return null; } return super.fold(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java index 638fa1b8db456..4f97bf60bd863 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java @@ -30,7 +30,7 @@ public Expression rule(Expression e) { // perform this early to prevent the rule from converting the null filter into nullifying the whole expression // P.S. this could be done inside the Aggregate but this place better centralizes the logic if (e instanceof AggregateFunction agg) { - if (Expressions.isNull(agg.filter())) { + if (Expressions.isGuaranteedNull(agg.filter())) { return agg.withFilter(Literal.of(agg.filter(), false)); } } @@ -38,13 +38,13 @@ public Expression rule(Expression e) { if (result != e) { return result; } else if (e instanceof In in) { - if (Expressions.isNull(in.value())) { + if (Expressions.isGuaranteedNull(in.value())) { return Literal.of(in, null); } } else if (e instanceof Alias == false && e.nullable() == Nullability.TRUE && e instanceof Categorize == false - && Expressions.anyMatch(e.children(), Expressions::isNull)) { + && Expressions.anyMatch(e.children(), Expressions::isGuaranteedNull)) { return Literal.of(e, null); } return e; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneFilters.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneFilters.java index b6f7ac9e464f4..00698d009ea23 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneFilters.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneFilters.java @@ -29,7 +29,7 @@ protected LogicalPlan rule(Filter filter) { if (TRUE.equals(condition)) { return filter.child(); } - if (FALSE.equals(condition) || Expressions.isNull(condition)) { + if (FALSE.equals(condition) || Expressions.isGuaranteedNull(condition)) { return PruneEmptyPlans.skipPlan(filter); } } @@ -42,8 +42,8 @@ protected LogicalPlan rule(Filter filter) { private static Expression foldBinaryLogic(BinaryLogic binaryLogic) { if (binaryLogic instanceof Or or) { - boolean nullLeft = Expressions.isNull(or.left()); - boolean nullRight = Expressions.isNull(or.right()); + boolean nullLeft = Expressions.isGuaranteedNull(or.left()); + boolean nullRight = Expressions.isGuaranteedNull(or.right()); if (nullLeft && nullRight) { return new Literal(binaryLogic.source(), null, DataType.NULL); } @@ -55,7 +55,7 @@ private static Expression foldBinaryLogic(BinaryLogic binaryLogic) { } } if (binaryLogic instanceof And and) { - if (Expressions.isNull(and.left()) || Expressions.isNull(and.right())) { + if (Expressions.isGuaranteedNull(and.left()) || Expressions.isGuaranteedNull(and.right())) { return new Literal(binaryLogic.source(), null, DataType.NULL); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java index 930b485dbd374..9e9ae6a9a559d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java @@ -30,7 +30,7 @@ public Expression rule(In in) { List foldables = new ArrayList<>(in.list().size()); List nonFoldables = new ArrayList<>(in.list().size()); in.list().forEach(e -> { - if (e.foldable() && Expressions.isNull(e) == false) { // keep `null`s, needed for the 3VL + if (e.foldable() && Expressions.isGuaranteedNull(e) == false) { // keep `null`s, needed for the 3VL foldables.add(e); } else { nonFoldables.add(e); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index b76781f76f4af..c2a26845d4e88 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -4820,7 +4820,7 @@ private static boolean oneLeaveIsNull(Expression e) { e.forEachUp(node -> { if (node.children().size() == 0) { - result.set(result.get() || Expressions.isNull(node)); + result.set(result.get() || Expressions.isGuaranteedNull(node)); } }); From ab6fcc4e749ae575e6505498d18de19ce3db4ca2 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 6 Dec 2024 18:50:04 +0100 Subject: [PATCH 088/204] Fix mocking in SyntheticSourceLicenseServiceTests (#118155) Some mock verifies where missing and `LicenseState#copyCurrentLicenseState(...)` wasn't always mocked. And because of incorrect mocking the testGoldOrPlatinumLicenseCustomCutoffDate() test had an incorrect assertion. --- .../SyntheticSourceLicenseServiceTests.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseServiceTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseServiceTests.java index 90a13b16c028e..0eb0d21ff2e78 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseServiceTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseServiceTests.java @@ -41,6 +41,7 @@ public void setup() throws Exception { public void testLicenseAllowsSyntheticSource() { MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(true); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); @@ -53,6 +54,7 @@ public void testLicenseAllowsSyntheticSource() { public void testLicenseAllowsSyntheticSourceTemplateValidation() { MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(true); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); @@ -65,6 +67,7 @@ public void testLicenseAllowsSyntheticSourceTemplateValidation() { public void testDefaultDisallow() { MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(false); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); @@ -77,6 +80,7 @@ public void testDefaultDisallow() { public void testFallback() { MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(true); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); @@ -95,6 +99,7 @@ public void testGoldOrPlatinumLicense() throws Exception { when(mockLicenseService.getLicense()).thenReturn(license); MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); when(licenseState.getOperationMode()).thenReturn(license.operationMode()); when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY))).thenReturn(true); licenseService.setLicenseState(licenseState); @@ -103,6 +108,8 @@ public void testGoldOrPlatinumLicense() throws Exception { "legacy licensed usage is allowed, so not fallback to stored source", licenseService.fallbackToStoredSource(false, true) ); + Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE)); + Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY)); Mockito.verify(licenseState, Mockito.times(1)).featureUsed(any()); } @@ -112,6 +119,7 @@ public void testGoldOrPlatinumLicenseLegacyLicenseNotAllowed() throws Exception when(mockLicenseService.getLicense()).thenReturn(license); MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); when(licenseState.getOperationMode()).thenReturn(license.operationMode()); when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(false); licenseService.setLicenseState(licenseState); @@ -125,14 +133,16 @@ public void testGoldOrPlatinumLicenseLegacyLicenseNotAllowed() throws Exception } public void testGoldOrPlatinumLicenseBeyondCutoffDate() throws Exception { - long start = LocalDateTime.of(2025, 1, 1, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); + long start = LocalDateTime.of(2025, 2, 5, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); License license = createGoldOrPlatinumLicense(start); mockLicenseService = mock(LicenseService.class); when(mockLicenseService.getLicense()).thenReturn(license); MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); when(licenseState.getOperationMode()).thenReturn(license.operationMode()); when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(false); + when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY))).thenReturn(true); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); assertTrue("beyond cutoff date, so fallback to stored source", licenseService.fallbackToStoredSource(false, true)); @@ -143,19 +153,21 @@ public void testGoldOrPlatinumLicenseBeyondCutoffDate() throws Exception { public void testGoldOrPlatinumLicenseCustomCutoffDate() throws Exception { licenseService = new SyntheticSourceLicenseService(Settings.EMPTY, "2025-01-02T00:00"); - long start = LocalDateTime.of(2025, 1, 1, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); + long start = LocalDateTime.of(2025, 1, 3, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); License license = createGoldOrPlatinumLicense(start); mockLicenseService = mock(LicenseService.class); when(mockLicenseService.getLicense()).thenReturn(license); MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); when(licenseState.getOperationMode()).thenReturn(license.operationMode()); + when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE))).thenReturn(false); when(licenseState.isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY))).thenReturn(true); licenseService.setLicenseState(licenseState); licenseService.setLicenseService(mockLicenseService); - assertFalse("custom cutoff date, so fallback to stored source", licenseService.fallbackToStoredSource(false, true)); - Mockito.verify(licenseState, Mockito.times(1)).featureUsed(any()); - Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE_LEGACY)); + assertTrue("custom cutoff date, so fallback to stored source", licenseService.fallbackToStoredSource(false, true)); + Mockito.verify(licenseState, Mockito.times(1)).isAllowed(same(SyntheticSourceLicenseService.SYNTHETIC_SOURCE_FEATURE)); + Mockito.verify(licenseState, Mockito.never()).featureUsed(any()); } static License createEnterpriseLicense() throws Exception { From 27aac9654d0b226ac5988f635858ae6404a75cda Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 7 Dec 2024 05:19:17 +1100 Subject: [PATCH 089/204] Mute org.elasticsearch.xpack.inference.DefaultEndPointsIT testInferDeploysDefaultRerank #118184 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 8f1030279efc9..887b462fa122e 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -266,6 +266,9 @@ tests: - class: org.elasticsearch.packaging.test.DebPreservationTests method: test40RestartOnUpgrade issue: https://github.com/elastic/elasticsearch/issues/118170 +- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT + method: testInferDeploysDefaultRerank + issue: https://github.com/elastic/elasticsearch/issues/118184 # Examples: # From c580024ea92fd561089380eb4078f576c47b72b9 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Fri, 6 Dec 2024 18:42:50 +0000 Subject: [PATCH 090/204] Add Highlighter for Semantic Text Fields (#118064) This PR introduces a new highlighter, `semantic`, tailored for semantic text fields. It extracts the most relevant fragments by scoring nested chunks using the original semantic query. In this initial version, the highlighter returns only the original chunks computed during ingestion. However, this is an implementation detail, and future enhancements could combine multiple chunks to generate the fragments. --- docs/changelog/118064.yaml | 5 + .../mapping/types/semantic-text.asciidoc | 55 +-- .../xpack/inference/InferenceFeatures.java | 5 +- .../xpack/inference/InferencePlugin.java | 7 + .../highlight/SemanticTextHighlighter.java | 226 +++++++++ .../inference/mapper/SemanticTextField.java | 2 +- .../mapper/SemanticTextFieldMapper.java | 20 +- .../SemanticTextHighlighterTests.java | 288 +++++++++++ .../mapper/SemanticTextFieldMapperTests.java | 8 +- .../queries/SemanticQueryBuilderTests.java | 10 +- .../xpack/inference/highlight/mappings.json | 27 + .../xpack/inference/highlight/queries.json | 467 ++++++++++++++++++ .../inference/highlight/sample-doc.json.gz | Bin 0 -> 388098 bytes .../90_semantic_text_highlighter.yml | 242 +++++++++ 14 files changed, 1314 insertions(+), 48 deletions(-) create mode 100644 docs/changelog/118064.yaml create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighter.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java create mode 100644 x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/mappings.json create mode 100644 x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/queries.json create mode 100644 x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/sample-doc.json.gz create mode 100644 x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/90_semantic_text_highlighter.yml diff --git a/docs/changelog/118064.yaml b/docs/changelog/118064.yaml new file mode 100644 index 0000000000000..7d12f365bf142 --- /dev/null +++ b/docs/changelog/118064.yaml @@ -0,0 +1,5 @@ +pr: 118064 +summary: Add Highlighter for Semantic Text Fields +area: Highlighting +type: feature +issues: [] diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc index f76a9352c2fe8..b3e103ec6dbd9 100644 --- a/docs/reference/mapping/types/semantic-text.asciidoc +++ b/docs/reference/mapping/types/semantic-text.asciidoc @@ -112,50 +112,43 @@ Trying to <> that is used on a {infer-cap} endpoints have a limit on the amount of text they can process. To allow for large amounts of text to be used in semantic search, `semantic_text` automatically generates smaller passages if needed, called _chunks_. -Each chunk will include the text subpassage and the corresponding embedding generated from it. +Each chunk refers to a passage of the text and the corresponding embedding generated from it. When querying, the individual passages will be automatically searched for each document, and the most relevant passage will be used to compute a score. For more details on chunking and how to configure chunking settings, see <> in the Inference API documentation. +Refer to <> to learn more about +semantic search using `semantic_text` and the `semantic` query. [discrete] -[[semantic-text-structure]] -==== `semantic_text` structure +[[semantic-text-highlighting]] +==== Extracting Relevant Fragments from Semantic Text -Once a document is ingested, a `semantic_text` field will have the following structure: +You can extract the most relevant fragments from a semantic text field by using the <> in the <>. -[source,console-result] +[source,console] ------------------------------------------------------------ -"inference_field": { - "text": "these are not the droids you're looking for", <1> - "inference": { - "inference_id": "my-elser-endpoint", <2> - "model_settings": { <3> - "task_type": "sparse_embedding" +PUT test-index +{ + "query": { + "semantic": { + "field": "my_semantic_field" + } }, - "chunks": [ <4> - { - "text": "these are not the droids you're looking for", - "embeddings": { - (...) + "highlight": { + "fields": { + "my_semantic_field": { + "type": "semantic", + "number_of_fragments": 2, <1> + "order": "score" <2> + } } - } - ] - } + } } ------------------------------------------------------------ -// TEST[skip:TBD] -<1> The field will become an object structure to accommodate both the original -text and the inference results. -<2> The `inference_id` used to generate the embeddings. -<3> Model settings, including the task type and dimensions/similarity if -applicable. -<4> Inference results will be grouped in chunks, each with its corresponding -text and embeddings. - -Refer to <> to learn more about -semantic search using `semantic_text` and the `semantic` query. - +// TEST[skip:Requires inference endpoint] +<1> Specifies the maximum number of fragments to return. +<2> Sorts highlighted fragments by score when set to `score`. By default, fragments will be output in the order they appear in the field (order: none). [discrete] [[custom-indexing]] diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index c82f287792a7c..67892dfe78624 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -33,6 +33,8 @@ public Set getFeatures() { ); } + private static final NodeFeature SEMANTIC_TEXT_HIGHLIGHTER = new NodeFeature("semantic_text.highlighter"); + @Override public Set getTestFeatures() { return Set.of( @@ -40,7 +42,8 @@ public Set getTestFeatures() { SemanticTextFieldMapper.SEMANTIC_TEXT_SINGLE_FIELD_UPDATE_FIX, SemanticTextFieldMapper.SEMANTIC_TEXT_DELETE_FIX, SemanticTextFieldMapper.SEMANTIC_TEXT_ZERO_SIZE_FIX, - SemanticTextFieldMapper.SEMANTIC_TEXT_ALWAYS_EMIT_INFERENCE_ID_FIX + SemanticTextFieldMapper.SEMANTIC_TEXT_ALWAYS_EMIT_INFERENCE_ID_FIX, + SEMANTIC_TEXT_HIGHLIGHTER ); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index 3c14e51a3c2d4..d7d623ab20143 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -37,6 +37,7 @@ import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.search.fetch.subphase.highlight.Highlighter; import org.elasticsearch.search.rank.RankBuilder; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.threadpool.ExecutorBuilder; @@ -67,6 +68,7 @@ import org.elasticsearch.xpack.inference.external.http.retry.RetrySettings; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.RequestExecutorServiceSettings; +import org.elasticsearch.xpack.inference.highlight.SemanticTextHighlighter; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import org.elasticsearch.xpack.inference.mapper.OffsetSourceFieldMapper; import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper; @@ -417,4 +419,9 @@ public List> getRetrievers() { new RetrieverSpec<>(new ParseField(RandomRankBuilder.NAME), RandomRankRetrieverBuilder::fromXContent) ); } + + @Override + public Map getHighlighters() { + return Map.of(SemanticTextHighlighter.NAME, new SemanticTextHighlighter()); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighter.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighter.java new file mode 100644 index 0000000000000..f2bfa72ec617a --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighter.java @@ -0,0 +1,226 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.highlight; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnByteVectorQuery; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.DenseVectorFieldType; +import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper.SparseVectorFieldType; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.fetch.subphase.highlight.FieldHighlightContext; +import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; +import org.elasticsearch.search.fetch.subphase.highlight.Highlighter; +import org.elasticsearch.search.vectors.VectorData; +import org.elasticsearch.xpack.core.ml.search.SparseVectorQueryWrapper; +import org.elasticsearch.xpack.inference.mapper.SemanticTextField; +import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * A {@link Highlighter} designed for the {@link SemanticTextFieldMapper}. + * This highlighter extracts semantic queries and evaluates them against each chunk produced by the semantic text field. + * It returns the top-scoring chunks as snippets, optionally sorted by their scores. + */ +public class SemanticTextHighlighter implements Highlighter { + public static final String NAME = "semantic"; + + private record OffsetAndScore(int offset, float score) {} + + @Override + public boolean canHighlight(MappedFieldType fieldType) { + if (fieldType instanceof SemanticTextFieldMapper.SemanticTextFieldType) { + return true; + } + return false; + } + + @Override + public HighlightField highlight(FieldHighlightContext fieldContext) throws IOException { + SemanticTextFieldMapper.SemanticTextFieldType fieldType = (SemanticTextFieldMapper.SemanticTextFieldType) fieldContext.fieldType; + if (fieldType.getEmbeddingsField() == null) { + // nothing indexed yet + return null; + } + + final List queries = switch (fieldType.getModelSettings().taskType()) { + case SPARSE_EMBEDDING -> extractSparseVectorQueries( + (SparseVectorFieldType) fieldType.getEmbeddingsField().fieldType(), + fieldContext.query + ); + case TEXT_EMBEDDING -> extractDenseVectorQueries( + (DenseVectorFieldType) fieldType.getEmbeddingsField().fieldType(), + fieldContext.query + ); + default -> throw new IllegalStateException( + "Wrong task type for a semantic text field, got [" + fieldType.getModelSettings().taskType().name() + "]" + ); + }; + if (queries.isEmpty()) { + // nothing to highlight + return null; + } + + int numberOfFragments = fieldContext.field.fieldOptions().numberOfFragments() <= 0 + ? 1 // we return the best fragment by default + : fieldContext.field.fieldOptions().numberOfFragments(); + + List chunks = extractOffsetAndScores( + fieldContext.context.getSearchExecutionContext(), + fieldContext.hitContext.reader(), + fieldType, + fieldContext.hitContext.docId(), + queries + ); + if (chunks.size() == 0) { + return null; + } + + chunks.sort(Comparator.comparingDouble(OffsetAndScore::score).reversed()); + int size = Math.min(chunks.size(), numberOfFragments); + if (fieldContext.field.fieldOptions().scoreOrdered() == false) { + chunks = chunks.subList(0, size); + chunks.sort(Comparator.comparingInt(c -> c.offset)); + } + Text[] snippets = new Text[size]; + List> nestedSources = XContentMapValues.extractNestedSources( + fieldType.getChunksField().fullPath(), + fieldContext.hitContext.source().source() + ); + for (int i = 0; i < size; i++) { + var chunk = chunks.get(i); + if (nestedSources.size() <= chunk.offset) { + throw new IllegalStateException( + String.format( + Locale.ROOT, + "Invalid content detected for field [%s]: the chunks size is [%d], " + + "but a reference to offset [%d] was found in the result.", + fieldType.name(), + nestedSources.size(), + chunk.offset + ) + ); + } + String content = (String) nestedSources.get(chunk.offset).get(SemanticTextField.CHUNKED_TEXT_FIELD); + if (content == null) { + throw new IllegalStateException( + String.format( + Locale.ROOT, + + "Invalid content detected for field [%s]: missing text for the chunk at offset [%d].", + fieldType.name(), + chunk.offset + ) + ); + } + snippets[i] = new Text(content); + } + return new HighlightField(fieldContext.fieldName, snippets); + } + + private List extractOffsetAndScores( + SearchExecutionContext context, + LeafReader reader, + SemanticTextFieldMapper.SemanticTextFieldType fieldType, + int docId, + List leafQueries + ) throws IOException { + var bitSet = context.bitsetFilter(fieldType.getChunksField().parentTypeFilter()).getBitSet(reader.getContext()); + int previousParent = docId > 0 ? bitSet.prevSetBit(docId - 1) : -1; + + BooleanQuery.Builder bq = new BooleanQuery.Builder().add(fieldType.getChunksField().nestedTypeFilter(), BooleanClause.Occur.FILTER); + leafQueries.stream().forEach(q -> bq.add(q, BooleanClause.Occur.SHOULD)); + Weight weight = new IndexSearcher(reader).createWeight(bq.build(), ScoreMode.COMPLETE, 1); + Scorer scorer = weight.scorer(reader.getContext()); + if (previousParent != -1) { + if (scorer.iterator().advance(previousParent) == DocIdSetIterator.NO_MORE_DOCS) { + return List.of(); + } + } else if (scorer.iterator().nextDoc() == DocIdSetIterator.NO_MORE_DOCS) { + return List.of(); + } + List results = new ArrayList<>(); + int offset = 0; + while (scorer.docID() < docId) { + results.add(new OffsetAndScore(offset++, scorer.score())); + if (scorer.iterator().nextDoc() == DocIdSetIterator.NO_MORE_DOCS) { + break; + } + } + return results; + } + + private List extractDenseVectorQueries(DenseVectorFieldType fieldType, Query querySection) { + // TODO: Handle knn section when semantic text field can be used. + List queries = new ArrayList<>(); + querySection.visit(new QueryVisitor() { + @Override + public boolean acceptField(String field) { + return fieldType.name().equals(field); + } + + @Override + public void consumeTerms(Query query, Term... terms) { + super.consumeTerms(query, terms); + } + + @Override + public void visitLeaf(Query query) { + if (query instanceof KnnFloatVectorQuery knnQuery) { + queries.add(fieldType.createExactKnnQuery(VectorData.fromFloats(knnQuery.getTargetCopy()), null)); + } else if (query instanceof KnnByteVectorQuery knnQuery) { + queries.add(fieldType.createExactKnnQuery(VectorData.fromBytes(knnQuery.getTargetCopy()), null)); + } + } + }); + return queries; + } + + private List extractSparseVectorQueries(SparseVectorFieldType fieldType, Query querySection) { + List queries = new ArrayList<>(); + querySection.visit(new QueryVisitor() { + @Override + public boolean acceptField(String field) { + return fieldType.name().equals(field); + } + + @Override + public void consumeTerms(Query query, Term... terms) { + super.consumeTerms(query, terms); + } + + @Override + public QueryVisitor getSubVisitor(BooleanClause.Occur occur, Query parent) { + if (parent instanceof SparseVectorQueryWrapper sparseVectorQuery) { + queries.add(sparseVectorQuery.getTermsQuery()); + } + return this; + } + }); + return queries; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java index e60e95b58770f..0f26f6577860f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java @@ -61,7 +61,7 @@ public record SemanticTextField(String fieldName, List originalValues, I static final String SEARCH_INFERENCE_ID_FIELD = "search_inference_id"; static final String CHUNKS_FIELD = "chunks"; static final String CHUNKED_EMBEDDINGS_FIELD = "embeddings"; - static final String CHUNKED_TEXT_FIELD = "text"; + public static final String CHUNKED_TEXT_FIELD = "text"; static final String MODEL_SETTINGS_FIELD = "model_settings"; static final String TASK_TYPE_FIELD = "task_type"; static final String DIMENSIONS_FIELD = "dimensions"; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 3744bf2a6dbed..683bb5a53028b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -46,7 +46,6 @@ import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.SimilarityMeasure; @@ -57,6 +56,7 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; +import org.elasticsearch.xpack.core.ml.search.SparseVectorQueryBuilder; import java.io.IOException; import java.util.ArrayList; @@ -529,17 +529,15 @@ public QueryBuilder semanticQuery(InferenceResults inferenceResults, Integer req ); } - // TODO: Use WeightedTokensQueryBuilder TextExpansionResults textExpansionResults = (TextExpansionResults) inferenceResults; - var boolQuery = QueryBuilders.boolQuery(); - for (var weightedToken : textExpansionResults.getWeightedTokens()) { - boolQuery.should( - QueryBuilders.termQuery(inferenceResultsFieldName, weightedToken.token()).boost(weightedToken.weight()) - ); - } - boolQuery.minimumShouldMatch(1); - - yield boolQuery; + yield new SparseVectorQueryBuilder( + inferenceResultsFieldName, + textExpansionResults.getWeightedTokens(), + null, + null, + null, + null + ); } case TEXT_EMBEDDING -> { if (inferenceResults instanceof MlTextEmbeddingResults == false) { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java new file mode 100644 index 0000000000000..7dc4d99e06acc --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java @@ -0,0 +1,288 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.highlight; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.join.ScoreMode; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.action.OriginalIndices; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.query.NestedQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.FetchContext; +import org.elasticsearch.search.fetch.FetchSubPhase; +import org.elasticsearch.search.fetch.subphase.highlight.FieldHighlightContext; +import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext; +import org.elasticsearch.search.internal.AliasFilter; +import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ml.search.SparseVectorQueryBuilder; +import org.elasticsearch.xpack.core.ml.search.WeightedToken; +import org.elasticsearch.xpack.inference.InferencePlugin; +import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper; +import org.junit.Before; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.GZIPInputStream; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.mockito.Mockito.mock; + +public class SemanticTextHighlighterTests extends MapperServiceTestCase { + private static final String SEMANTIC_FIELD_E5 = "body-e5"; + private static final String SEMANTIC_FIELD_ELSER = "body-elser"; + + private Map queries; + + @Override + protected Collection getPlugins() { + return List.of(new InferencePlugin(Settings.EMPTY)); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + var input = Streams.readFully(SemanticTextHighlighterTests.class.getResourceAsStream("queries.json")); + this.queries = XContentHelper.convertToMap(input, false, XContentType.JSON).v2(); + } + + @SuppressWarnings("unchecked") + public void testDenseVector() throws Exception { + var mapperService = createDefaultMapperService(); + Map queryMap = (Map) queries.get("dense_vector_1"); + float[] vector = readDenseVector(queryMap.get("embeddings")); + var fieldType = (SemanticTextFieldMapper.SemanticTextFieldType) mapperService.mappingLookup().getFieldType(SEMANTIC_FIELD_E5); + KnnVectorQueryBuilder knnQuery = new KnnVectorQueryBuilder(fieldType.getEmbeddingsField().fullPath(), vector, 10, 10, null); + NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(fieldType.getChunksField().fullPath(), knnQuery, ScoreMode.Max); + var shardRequest = createShardSearchRequest(nestedQueryBuilder); + var sourceToParse = new SourceToParse("0", readSampleDoc("sample-doc.json.gz"), XContentType.JSON); + + String[] expectedScorePassages = ((List) queryMap.get("expected_by_score")).toArray(String[]::new); + for (int i = 0; i < expectedScorePassages.length; i++) { + assertHighlightOneDoc( + mapperService, + shardRequest, + sourceToParse, + SEMANTIC_FIELD_E5, + i + 1, + HighlightBuilder.Order.SCORE, + Arrays.copyOfRange(expectedScorePassages, 0, i + 1) + ); + } + + String[] expectedOffsetPassages = ((List) queryMap.get("expected_by_offset")).toArray(String[]::new); + assertHighlightOneDoc( + mapperService, + shardRequest, + sourceToParse, + SEMANTIC_FIELD_E5, + expectedOffsetPassages.length, + HighlightBuilder.Order.NONE, + expectedOffsetPassages + ); + } + + @SuppressWarnings("unchecked") + public void testSparseVector() throws Exception { + var mapperService = createDefaultMapperService(); + Map queryMap = (Map) queries.get("sparse_vector_1"); + List tokens = readSparseVector(queryMap.get("embeddings")); + var fieldType = (SemanticTextFieldMapper.SemanticTextFieldType) mapperService.mappingLookup().getFieldType(SEMANTIC_FIELD_ELSER); + SparseVectorQueryBuilder sparseQuery = new SparseVectorQueryBuilder( + fieldType.getEmbeddingsField().fullPath(), + tokens, + null, + null, + null, + null + ); + NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(fieldType.getChunksField().fullPath(), sparseQuery, ScoreMode.Max); + var shardRequest = createShardSearchRequest(nestedQueryBuilder); + var sourceToParse = new SourceToParse("0", readSampleDoc("sample-doc.json.gz"), XContentType.JSON); + + String[] expectedScorePassages = ((List) queryMap.get("expected_by_score")).toArray(String[]::new); + for (int i = 0; i < expectedScorePassages.length; i++) { + assertHighlightOneDoc( + mapperService, + shardRequest, + sourceToParse, + SEMANTIC_FIELD_ELSER, + i + 1, + HighlightBuilder.Order.SCORE, + Arrays.copyOfRange(expectedScorePassages, 0, i + 1) + ); + } + + String[] expectedOffsetPassages = ((List) queryMap.get("expected_by_offset")).toArray(String[]::new); + assertHighlightOneDoc( + mapperService, + shardRequest, + sourceToParse, + SEMANTIC_FIELD_ELSER, + expectedOffsetPassages.length, + HighlightBuilder.Order.NONE, + expectedOffsetPassages + ); + } + + private MapperService createDefaultMapperService() throws IOException { + var mappings = Streams.readFully(SemanticTextHighlighterTests.class.getResourceAsStream("mappings.json")); + return createMapperService(mappings.utf8ToString()); + } + + private float[] readDenseVector(Object value) { + if (value instanceof List lst) { + float[] res = new float[lst.size()]; + int pos = 0; + for (var obj : lst) { + if (obj instanceof Number number) { + res[pos++] = number.floatValue(); + } else { + throw new IllegalArgumentException("Expected number, got " + obj.getClass().getSimpleName()); + } + } + return res; + } + throw new IllegalArgumentException("Expected list, got " + value.getClass().getSimpleName()); + } + + private List readSparseVector(Object value) { + if (value instanceof Map map) { + List res = new ArrayList<>(); + for (var entry : map.entrySet()) { + if (entry.getValue() instanceof Number number) { + res.add(new WeightedToken((String) entry.getKey(), number.floatValue())); + } else { + throw new IllegalArgumentException("Expected number, got " + entry.getValue().getClass().getSimpleName()); + } + } + return res; + } + throw new IllegalArgumentException("Expected map, got " + value.getClass().getSimpleName()); + } + + private void assertHighlightOneDoc( + MapperService mapperService, + ShardSearchRequest request, + SourceToParse source, + String fieldName, + int numFragments, + HighlightBuilder.Order order, + String[] expectedPassages + ) throws Exception { + SemanticTextFieldMapper fieldMapper = (SemanticTextFieldMapper) mapperService.mappingLookup().getMapper(fieldName); + var doc = mapperService.documentMapper().parse(source); + assertNull(doc.dynamicMappingsUpdate()); + try (Directory dir = newDirectory()) { + IndexWriterConfig iwc = newIndexWriterConfig(new StandardAnalyzer()); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc); + iw.addDocuments(doc.docs()); + try (DirectoryReader reader = wrapInMockESDirectoryReader(iw.getReader())) { + IndexSearcher searcher = newSearcher(reader); + iw.close(); + TopDocs topDocs = searcher.search(Queries.newNonNestedFilter(IndexVersion.current()), 1, Sort.INDEXORDER); + assertThat(topDocs.totalHits.value(), equalTo(1L)); + int docID = topDocs.scoreDocs[0].doc; + SemanticTextHighlighter highlighter = new SemanticTextHighlighter(); + var execContext = createSearchExecutionContext(mapperService); + var luceneQuery = execContext.toQuery(request.source().query()).query(); + FetchContext fetchContext = mock(FetchContext.class); + Mockito.when(fetchContext.highlight()).thenReturn(new SearchHighlightContext(Collections.emptyList())); + Mockito.when(fetchContext.query()).thenReturn(luceneQuery); + Mockito.when(fetchContext.getSearchExecutionContext()).thenReturn(execContext); + + FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext( + new SearchHit(docID), + getOnlyLeafReader(reader).getContext(), + docID, + Map.of(), + Source.fromBytes(source.source()), + new RankDoc(docID, Float.NaN, 0) + ); + try { + var highlightContext = new HighlightBuilder().field(fieldName, 0, numFragments) + .order(order) + .highlighterType(SemanticTextHighlighter.NAME) + .build(execContext); + + for (var fieldContext : highlightContext.fields()) { + FieldHighlightContext context = new FieldHighlightContext( + fieldName, + fieldContext, + fieldMapper.fieldType(), + fetchContext, + hitContext, + luceneQuery, + new HashMap<>() + ); + var result = highlighter.highlight(context); + assertThat(result.fragments().length, equalTo(expectedPassages.length)); + for (int i = 0; i < result.fragments().length; i++) { + assertThat(result.fragments()[i].string(), equalTo(expectedPassages[i])); + } + } + } finally { + hitContext.hit().decRef(); + } + } + } + } + + private SearchRequest createSearchRequest(QueryBuilder queryBuilder) { + SearchRequest request = new SearchRequest(); + request.source(new SearchSourceBuilder()); + request.allowPartialSearchResults(false); + request.source().query(queryBuilder); + return request; + } + + private ShardSearchRequest createShardSearchRequest(QueryBuilder queryBuilder) { + SearchRequest request = createSearchRequest(queryBuilder); + return new ShardSearchRequest(OriginalIndices.NONE, request, new ShardId("index", "index", 0), 0, 1, AliasFilter.EMPTY, 1, 0, null); + } + + private BytesReference readSampleDoc(String fileName) throws IOException { + try (var in = new GZIPInputStream(SemanticTextHighlighterTests.class.getResourceAsStream(fileName))) { + return new BytesArray(new BytesRef(in.readAllBytes())); + } + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index fd60d9687f437..c6a492dfcf4e9 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -61,6 +61,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.ml.search.SparseVectorQueryWrapper; import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.model.TestModel; import org.junit.AssumptionViolatedException; @@ -1110,7 +1111,12 @@ private static Query generateNestedTermSparseVectorQuery(NestedLookup nestedLook } queryBuilder.add(new BooleanClause(mapper.nestedTypeFilter(), BooleanClause.Occur.FILTER)); - return new ESToParentBlockJoinQuery(queryBuilder.build(), parentFilter, ScoreMode.Total, null); + return new ESToParentBlockJoinQuery( + new SparseVectorQueryWrapper(fieldName, queryBuilder.build()), + parentFilter, + ScoreMode.Total, + null + ); } private static void assertChildLeafNestedDocument( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java index b8bcb766b53e1..36aa2200eceae 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java @@ -45,12 +45,14 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.XPackClientPlugin; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; +import org.elasticsearch.xpack.core.ml.search.SparseVectorQueryWrapper; import org.elasticsearch.xpack.core.ml.search.WeightedToken; import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.mapper.SemanticTextField; @@ -114,7 +116,7 @@ public void setUp() throws Exception { @Override protected Collection> getPlugins() { - return List.of(InferencePlugin.class, FakeMlPlugin.class); + return List.of(XPackClientPlugin.class, InferencePlugin.class, FakeMlPlugin.class); } @Override @@ -194,9 +196,11 @@ protected void doAssertLuceneQuery(SemanticQueryBuilder queryBuilder, Query quer private void assertSparseEmbeddingLuceneQuery(Query query) { Query innerQuery = assertOuterBooleanQuery(query); - assertThat(innerQuery, instanceOf(BooleanQuery.class)); + assertThat(innerQuery, instanceOf(SparseVectorQueryWrapper.class)); + var sparseQuery = (SparseVectorQueryWrapper) innerQuery; + assertThat(((SparseVectorQueryWrapper) innerQuery).getTermsQuery(), instanceOf(BooleanQuery.class)); - BooleanQuery innerBooleanQuery = (BooleanQuery) innerQuery; + BooleanQuery innerBooleanQuery = (BooleanQuery) sparseQuery.getTermsQuery(); assertThat(innerBooleanQuery.clauses().size(), equalTo(queryTokenCount)); innerBooleanQuery.forEach(c -> { assertThat(c.occur(), equalTo(SHOULD)); diff --git a/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/mappings.json b/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/mappings.json new file mode 100644 index 0000000000000..9841ee0aed6e2 --- /dev/null +++ b/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/mappings.json @@ -0,0 +1,27 @@ +{ + "_doc": { + "properties": { + "body": { + "type": "text", + "copy_to": ["body-elser", "body-e5"] + }, + "body-e5": { + "type": "semantic_text", + "inference_id": ".multilingual-e5-small-elasticsearch", + "model_settings": { + "task_type": "text_embedding", + "dimensions": 384, + "similarity": "cosine", + "element_type": "float" + } + }, + "body-elser": { + "type": "semantic_text", + "inference_id": ".elser-2-elasticsearch", + "model_settings": { + "task_type": "sparse_embedding" + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/queries.json b/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/queries.json new file mode 100644 index 0000000000000..6227f3f498854 --- /dev/null +++ b/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/queries.json @@ -0,0 +1,467 @@ +{ + "dense_vector_1": { + "embeddings": [ + 0.09475211, + 0.044564713, + -0.04378501, + -0.07908551, + 0.04332011, + -0.03891992, + -0.0062305215, + 0.024245035, + -0.008976331, + 0.032832284, + 0.052760173, + 0.008123907, + 0.09049037, + -0.01637332, + -0.054353267, + 0.00771307, + 0.08545496, + -0.079716265, + -0.045666866, + -0.04369993, + 0.009189822, + -0.013782891, + -0.07701858, + 0.037278354, + 0.049807206, + 0.078036495, + -0.059533164, + 0.051413406, + 0.040234447, + -0.038139492, + -0.085189626, + -0.045546446, + 0.0544375, + -0.05604156, + 0.057408098, + 0.041913517, + -0.037348013, + -0.025998272, + 0.08486864, + -0.046678443, + 0.0041820924, + 0.007514462, + 0.06424746, + 0.044233218, + 0.103267275, + 0.014130771, + -0.049954403, + 0.04226959, + -0.08346965, + -0.01639249, + -0.060537644, + 0.04546336, + 0.012866155, + 0.05375096, + 0.036775924, + -0.0762226, + -0.037304543, + -0.05692274, + -0.055807598, + 0.0040082196, + 0.059259634, + 0.012022011, + -8.0863154E-4, + 0.0070405705, + 0.050255686, + 0.06810016, + 0.017190414, + 0.051975194, + -0.051436286, + 0.023408439, + -0.029802637, + 0.034137156, + -0.004660689, + -0.0442122, + 0.019065322, + 0.030806554, + 0.0064652697, + -0.066789865, + 0.057111286, + 0.009412479, + -0.041444767, + -0.06807582, + -0.085881524, + 0.04901128, + -0.047871742, + 0.06328623, + 0.040418074, + -0.081432894, + 0.058384005, + 0.006206527, + 0.045801315, + 0.037274595, + -0.054337103, + -0.06755516, + -0.07396888, + -0.043732334, + -0.052053086, + 0.03210978, + 0.048101492, + -0.083828256, + 0.05205026, + -0.048474856, + 0.029116616, + -0.10924888, + 0.003796487, + 0.030567763, + 0.026949523, + -0.052353345, + 0.043198872, + -0.09456988, + -0.05711594, + -2.2292069E-4, + 0.032972734, + 0.054394923, + -0.0767535, + -0.02710579, + -0.032135617, + -0.01732382, + 0.059442326, + -0.07686165, + 0.07104082, + -0.03090021, + -0.05450075, + -0.038997203, + -0.07045443, + 0.00483161, + 0.010933604, + 0.020874644, + 0.037941266, + 0.019729063, + 0.06178368, + 0.013503478, + -0.008584046, + 0.045592044, + 0.05528768, + 0.11568184, + 0.0041300594, + 0.015404516, + -3.8067883E-4, + -0.06365399, + -0.07826643, + 0.061575573, + -0.060548335, + 0.05706082, + 0.042301804, + 0.052173313, + 0.07193179, + -0.03839231, + 0.0734415, + -0.045380164, + 0.02832276, + 0.003745178, + 0.058844633, + 0.04307504, + 0.037800383, + -0.031050054, + -0.06856359, + -0.059114788, + -0.02148857, + 0.07854358, + -0.03253363, + -0.04566468, + -0.019933948, + -0.057993464, + -0.08677458, + -0.06626883, + 0.031657256, + 0.101128764, + -0.08050056, + -0.050226066, + -0.014335166, + 0.050344367, + -0.06851419, + 0.008698909, + -0.011893435, + 0.07741272, + -0.059579294, + 0.03250109, + 0.058700256, + 0.046834726, + -0.035081457, + -0.0043140925, + -0.09764087, + -0.0034994273, + -0.034056358, + -0.019066337, + -0.034376107, + 0.012964423, + 0.029291175, + -0.012090671, + 0.021585712, + 0.028859599, + -0.04391145, + -0.071166754, + -0.031040335, + 0.02808108, + -0.05621317, + 0.06543945, + 0.10094665, + 0.041057374, + -0.03222324, + -0.063366964, + 0.064944476, + 0.023641933, + 0.06806713, + 0.06806097, + -0.08220105, + 0.04148528, + -0.09254079, + 0.044620737, + 0.05526614, + -0.03849534, + -0.04722273, + 0.0670776, + -0.024274077, + -0.016903497, + 0.07584147, + 0.04760533, + -0.038843267, + -0.028365409, + 0.08022705, + -0.039916333, + 0.049067073, + -0.030701574, + -0.057169467, + 0.043025102, + 0.07109674, + -0.047296863, + -0.047463104, + 0.040868305, + -0.04409507, + -0.034977127, + -0.057109762, + -0.08616165, + -0.03486079, + -0.046201482, + 0.025963873, + 0.023392359, + 0.09594902, + -0.007847159, + -0.021231368, + 0.009007263, + 0.0032713825, + -0.06876065, + 0.03169641, + -7.2582875E-4, + -0.07049708, + 0.03900843, + -0.0075472407, + 0.05184822, + 0.06452079, + -0.09832754, + -0.012775799, + -0.03925948, + -0.029761659, + 0.0065437574, + 0.0815465, + 0.0411695, + -0.0702844, + -0.009533786, + 0.07024532, + 0.0098710675, + 0.09915362, + 0.0415453, + 0.050641853, + 0.047463298, + -0.058609713, + -0.029499197, + -0.05100956, + -0.03441709, + -0.06348122, + 0.014784361, + 0.056317374, + -0.10280704, + -0.04008354, + -0.018926824, + 0.08832836, + 0.124804, + -0.047645308, + -0.07122146, + -9.886527E-4, + 0.03850324, + 0.048501793, + 0.07072816, + 0.06566776, + -0.013678872, + 0.010010848, + 0.06483413, + -0.030036367, + -0.029748922, + -0.007482364, + -0.05180385, + 0.03698522, + -0.045453787, + 0.056604166, + 0.029394176, + 0.028589265, + -0.012185886, + -0.06919616, + 0.0711641, + -0.034055933, + -0.053101335, + 0.062319, + 0.021600349, + -0.038718067, + 0.060814686, + 0.05087301, + -0.020297311, + 0.016493896, + 0.032162152, + 0.046740912, + 0.05461355, + -0.07024665, + 0.025609337, + -0.02504801, + 0.06765588, + -0.032994855, + -0.037897404, + -0.045783922, + -0.05689299, + -0.040437017, + -0.07904339, + -0.031415287, + -0.029216278, + 0.017395392, + 0.03449264, + -0.025653394, + -0.06283088, + 0.049027324, + 0.016229525, + -0.00985347, + -0.053974394, + -0.030257035, + 0.04325515, + -0.012293731, + -0.002446129, + -0.05567076, + 0.06374684, + -0.03153897, + -0.04475149, + 0.018582936, + 0.025716115, + -0.061778374, + 0.04196277, + -0.04134671, + -0.07396272, + 0.05846184, + 0.006558759, + -0.09745666, + 0.07587805, + 0.0137483915, + -0.100933895, + 0.032008193, + 0.04293283, + 0.017870268, + 0.032806385, + -0.0635923, + -0.019672254, + 0.022225974, + 0.04304554, + -0.06043949, + -0.0285274, + 0.050868835, + 0.057003833, + 0.05740866, + 0.020068677, + -0.034312245, + -0.021671802, + 0.014769731, + -0.07328285, + -0.009586734, + 0.036420938, + -0.022188472, + -0.008200541, + -0.010765854, + -0.06949713, + -0.07555878, + 0.045306854, + -0.05424466, + -0.03647476, + 0.06266633, + 0.08346125, + 0.060288202, + 0.0548457 + ], + "expected_by_score": [ + "The ancient oppidum that corresponds to the modern city of Paris was first mentioned in the mid-1st century BC by Julius Caesar as Luteciam Parisiorum ('Lutetia of the Parisii') and is later attested as Parision in the 5th century AD, then as Paris in 1265. During the Roman period, it was commonly known as Lutetia or Lutecia in Latin, and as Leukotekía in Greek, which is interpreted as either stemming from the Celtic root *lukot- ('mouse'), or from *luto- ('marsh, swamp').\n\n\nThe name Paris is derived from its early inhabitants, the Parisii, a Gallic tribe from the Iron Age and the Roman period. The meaning of the Gaulish ethnonym remains debated. According to Xavier Delamarre, it may derive from the Celtic root pario- ('cauldron'). Alfred Holder interpreted the name as 'the makers' or 'the commanders', by comparing it to the Welsh peryff ('lord, commander'), both possibly descending from a Proto-Celtic form reconstructed as *kwar-is-io-. Alternatively, Pierre-Yves Lambert proposed to translate Parisii as the 'spear people', by connecting the first element to the Old Irish carr ('spear'), derived from an earlier *kwar-sā. In any case, the city's name is not related to the Paris of Greek mythology.\n\n\nResidents of the city are known in English as Parisians and in French as Parisiens ( ⓘ). They are also pejoratively called Parigots ( ⓘ).\n\n\nHistory\n\nOrigins\n\n", + "After the marshland between the river Seine and its slower 'dead arm' to its north was filled in from around the 10th century, Paris's cultural centre began to move to the Right Bank. In 1137, a new city marketplace (today's Les Halles) replaced the two smaller ones on the Île de la Cité and Place de Grève (Place de l'Hôtel de Ville). The latter location housed the headquarters of Paris's river trade corporation, an organisation that later became, unofficially (although formally in later years), Paris's first municipal government.\n\n\nIn the late 12th century, Philip Augustus extended the Louvre fortress to defend the city against river invasions from the west, gave the city its first walls between 1190 and 1215, rebuilt its bridges to either side of its central island, and paved its main thoroughfares. In 1190, he transformed Paris's former cathedral school into a student-teacher corporation that would become the University of Paris and would draw students from all of Europe.\n\n\nWith 200,000 inhabitants in 1328, Paris, then already the capital of France, was the most populous city of Europe. By comparison, London in 1300 had 80,000 inhabitants. By the early fourteenth century, so much filth had collected inside urban Europe that French and Italian cities were naming streets after human waste. In medieval Paris, several street names were inspired by merde, the French word for \"shit\".\n\n\n", + "In March 2001, Bertrand Delanoë became the first socialist mayor. He was re-elected in March 2008. In 2007, in an effort to reduce car traffic, he introduced the Vélib', a system which rents bicycles. Bertrand Delanoë also transformed a section of the highway along the Left Bank of the Seine into an urban promenade and park, the Promenade des Berges de la Seine, which he inaugurated in June 2013.\n\n\nIn 2007, President Nicolas Sarkozy launched the Grand Paris project, to integrate Paris more closely with the towns in the region around it. After many modifications, the new area, named the Metropolis of Grand Paris, with a population of 6.7 million, was created on 1 January 2016. In 2011, the City of Paris and the national government approved the plans for the Grand Paris Express, totalling 205 km (127 mi) of automated metro lines to connect Paris, the innermost three departments around Paris, airports and high-speed rail (TGV) stations, at an estimated cost of €35 billion. The system is scheduled to be completed by 2030.\n\n\nIn January 2015, Al-Qaeda in the Arabian Peninsula claimed attacks across the Paris region. 1.5 million people marched in Paris in a show of solidarity against terrorism and in support of freedom of speech. In November of the same year, terrorist attacks, claimed by ISIL, killed 130 people and injured more than 350.\n\n\n", + "\nParis (.mw-parser-output .IPA-label-small{font-size:85%}.mw-parser-output .references .IPA-label-small,.mw-parser-output .infobox .IPA-label-small,.mw-parser-output .navbox .IPA-label-small{font-size:100%}French pronunciation: ⓘ) is the capital and largest city of France. With an estimated population of 2,102,650 residents in January 2023 in an area of more than 105 km2 (41 sq mi), Paris is the fourth-largest city in the European Union and the 30th most densely populated city in the world in 2022. Since the 17th century, Paris has been one of the world's major centres of finance, diplomacy, commerce, culture, fashion, and gastronomy. Because of its leading role in the arts and sciences and its early adaptation of extensive street lighting, it became known as the City of Light in the 19th century.\n\n\nThe City of Paris is the centre of the Île-de-France region, or Paris Region, with an official estimated population of 12,271,794 inhabitants in January 2023, or about 19% of the population of France. The Paris Region had a nominal GDP of €765 billion (US$1.064 trillion when adjusted for PPP) in 2021, the highest in the European Union. According to the Economist Intelligence Unit Worldwide Cost of Living Survey, in 2022, Paris was the city with the ninth-highest cost of living in the world.\n\n\n", + "Bal-musette is a style of French music and dance that first became popular in Paris in the 1870s and 1880s; by 1880 Paris had some 150 dance halls. Patrons danced the bourrée to the accompaniment of the cabrette (a bellows-blown bagpipe locally called a \"musette\") and often the vielle à roue (hurdy-gurdy) in the cafés and bars of the city. Parisian and Italian musicians who played the accordion adopted the style and established themselves in Auvergnat bars, and Paris became a major centre for jazz and still attracts jazz musicians from all around the world to its clubs and cafés.\n\n\nParis is the spiritual home of gypsy jazz in particular, and many of the Parisian jazzmen who developed in the first half of the 20th century began by playing Bal-musette in the city. Django Reinhardt rose to fame in Paris, having moved to the 18th arrondissement in a caravan as a young boy, and performed with violinist Stéphane Grappelli and their Quintette du Hot Club de France in the 1930s and 1940s.\n\n\nImmediately after the War the Saint-Germain-des-Pres quarter and the nearby Saint-Michel quarter became home to many small jazz clubs, including the Caveau des Lorientais, the Club Saint-Germain, the Rose Rouge, the Vieux-Colombier, and the most famous, Le Tabou. They introduced Parisians to the music of Claude Luter, Boris Vian, Sydney Bechet, Mezz Mezzrow, and Henri Salvador. " + ], + "expected_by_offset": [ + "\nParis (.mw-parser-output .IPA-label-small{font-size:85%}.mw-parser-output .references .IPA-label-small,.mw-parser-output .infobox .IPA-label-small,.mw-parser-output .navbox .IPA-label-small{font-size:100%}French pronunciation: ⓘ) is the capital and largest city of France. With an estimated population of 2,102,650 residents in January 2023 in an area of more than 105 km2 (41 sq mi), Paris is the fourth-largest city in the European Union and the 30th most densely populated city in the world in 2022. Since the 17th century, Paris has been one of the world's major centres of finance, diplomacy, commerce, culture, fashion, and gastronomy. Because of its leading role in the arts and sciences and its early adaptation of extensive street lighting, it became known as the City of Light in the 19th century.\n\n\nThe City of Paris is the centre of the Île-de-France region, or Paris Region, with an official estimated population of 12,271,794 inhabitants in January 2023, or about 19% of the population of France. The Paris Region had a nominal GDP of €765 billion (US$1.064 trillion when adjusted for PPP) in 2021, the highest in the European Union. According to the Economist Intelligence Unit Worldwide Cost of Living Survey, in 2022, Paris was the city with the ninth-highest cost of living in the world.\n\n\n", + "The ancient oppidum that corresponds to the modern city of Paris was first mentioned in the mid-1st century BC by Julius Caesar as Luteciam Parisiorum ('Lutetia of the Parisii') and is later attested as Parision in the 5th century AD, then as Paris in 1265. During the Roman period, it was commonly known as Lutetia or Lutecia in Latin, and as Leukotekía in Greek, which is interpreted as either stemming from the Celtic root *lukot- ('mouse'), or from *luto- ('marsh, swamp').\n\n\nThe name Paris is derived from its early inhabitants, the Parisii, a Gallic tribe from the Iron Age and the Roman period. The meaning of the Gaulish ethnonym remains debated. According to Xavier Delamarre, it may derive from the Celtic root pario- ('cauldron'). Alfred Holder interpreted the name as 'the makers' or 'the commanders', by comparing it to the Welsh peryff ('lord, commander'), both possibly descending from a Proto-Celtic form reconstructed as *kwar-is-io-. Alternatively, Pierre-Yves Lambert proposed to translate Parisii as the 'spear people', by connecting the first element to the Old Irish carr ('spear'), derived from an earlier *kwar-sā. In any case, the city's name is not related to the Paris of Greek mythology.\n\n\nResidents of the city are known in English as Parisians and in French as Parisiens ( ⓘ). They are also pejoratively called Parigots ( ⓘ).\n\n\nHistory\n\nOrigins\n\n", + "After the marshland between the river Seine and its slower 'dead arm' to its north was filled in from around the 10th century, Paris's cultural centre began to move to the Right Bank. In 1137, a new city marketplace (today's Les Halles) replaced the two smaller ones on the Île de la Cité and Place de Grève (Place de l'Hôtel de Ville). The latter location housed the headquarters of Paris's river trade corporation, an organisation that later became, unofficially (although formally in later years), Paris's first municipal government.\n\n\nIn the late 12th century, Philip Augustus extended the Louvre fortress to defend the city against river invasions from the west, gave the city its first walls between 1190 and 1215, rebuilt its bridges to either side of its central island, and paved its main thoroughfares. In 1190, he transformed Paris's former cathedral school into a student-teacher corporation that would become the University of Paris and would draw students from all of Europe.\n\n\nWith 200,000 inhabitants in 1328, Paris, then already the capital of France, was the most populous city of Europe. By comparison, London in 1300 had 80,000 inhabitants. By the early fourteenth century, so much filth had collected inside urban Europe that French and Italian cities were naming streets after human waste. In medieval Paris, several street names were inspired by merde, the French word for \"shit\".\n\n\n", + "In March 2001, Bertrand Delanoë became the first socialist mayor. He was re-elected in March 2008. In 2007, in an effort to reduce car traffic, he introduced the Vélib', a system which rents bicycles. Bertrand Delanoë also transformed a section of the highway along the Left Bank of the Seine into an urban promenade and park, the Promenade des Berges de la Seine, which he inaugurated in June 2013.\n\n\nIn 2007, President Nicolas Sarkozy launched the Grand Paris project, to integrate Paris more closely with the towns in the region around it. After many modifications, the new area, named the Metropolis of Grand Paris, with a population of 6.7 million, was created on 1 January 2016. In 2011, the City of Paris and the national government approved the plans for the Grand Paris Express, totalling 205 km (127 mi) of automated metro lines to connect Paris, the innermost three departments around Paris, airports and high-speed rail (TGV) stations, at an estimated cost of €35 billion. The system is scheduled to be completed by 2030.\n\n\nIn January 2015, Al-Qaeda in the Arabian Peninsula claimed attacks across the Paris region. 1.5 million people marched in Paris in a show of solidarity against terrorism and in support of freedom of speech. In November of the same year, terrorist attacks, claimed by ISIL, killed 130 people and injured more than 350.\n\n\n", + "Bal-musette is a style of French music and dance that first became popular in Paris in the 1870s and 1880s; by 1880 Paris had some 150 dance halls. Patrons danced the bourrée to the accompaniment of the cabrette (a bellows-blown bagpipe locally called a \"musette\") and often the vielle à roue (hurdy-gurdy) in the cafés and bars of the city. Parisian and Italian musicians who played the accordion adopted the style and established themselves in Auvergnat bars, and Paris became a major centre for jazz and still attracts jazz musicians from all around the world to its clubs and cafés.\n\n\nParis is the spiritual home of gypsy jazz in particular, and many of the Parisian jazzmen who developed in the first half of the 20th century began by playing Bal-musette in the city. Django Reinhardt rose to fame in Paris, having moved to the 18th arrondissement in a caravan as a young boy, and performed with violinist Stéphane Grappelli and their Quintette du Hot Club de France in the 1930s and 1940s.\n\n\nImmediately after the War the Saint-Germain-des-Pres quarter and the nearby Saint-Michel quarter became home to many small jazz clubs, including the Caveau des Lorientais, the Club Saint-Germain, the Rose Rouge, the Vieux-Colombier, and the most famous, Le Tabou. They introduced Parisians to the music of Claude Luter, Boris Vian, Sydney Bechet, Mezz Mezzrow, and Henri Salvador. " + ] + }, + "sparse_vector_1": { + "embeddings": { + "paris": 2.9709616, + "date": 2.1960778, + "founded": 2.0555024, + "foundation": 1.412623, + "early": 1.2162757, + "founder": 1.1271698, + "french": 0.9213378, + "france": 0.86253893, + "city": 0.82978916, + "founding": 0.79722786, + "established": 0.7967043, + "ancient": 0.7392465, + "when": 0.71705, + "built": 0.6977878, + "treaty": 0.6846069, + "created": 0.68127465, + "century": 0.58926934, + "for": 0.55019474, + "was": 0.52475905, + "origin": 0.48785052, + "expedition": 0.48757303, + "history": 0.47960007, + "mint": 0.47878903, + "historical": 0.4714338, + "capital": 0.42984143, + "timeline": 0.4222377, + "colony": 0.3876187, + "tower": 0.3474891, + "medieval": 0.3272666, + "geography": 0.32456368, + "colonial": 0.30613664, + "location": 0.29013386, + "francisco": 0.22840048, + "orleans": 0.21971667, + "earlier": 0.20318772, + "jackson": 0.18424438, + "exact": 0.17109296, + "rome": 0.16320735, + "civilization": 0.15931238, + "spanish": 0.12759624, + "museum": 0.113024555, + "latin": 0.11201205, + "european": 0.10277243, + "architect": 0.0796932, + "united": 0.031233707 + }, + "expected_by_score": [ + "Clovis the Frank, the first king of the Merovingian dynasty, made the city his capital from 508. As the Frankish domination of Gaul began, there was a gradual immigration by the Franks to Paris and the Parisian Francien dialects were born. Fortification of the Île de la Cité failed to avert sacking by Vikings in 845, but Paris's strategic importance—with its bridges preventing ships from passing—was established by successful defence in the Siege of Paris (885–886), for which the then Count of Paris (comte de Paris), Odo of France, was elected king of West Francia. From the Capetian dynasty that began with the 987 election of Hugh Capet, Count of Paris and Duke of the Franks (duc des Francs), as king of a unified West Francia, Paris gradually became the largest and most prosperous city in France.\n\n\nHigh and Late Middle Ages to Louis XIV\n\nBy the end of the 12th century, Paris had become the political, economic, religious, and cultural capital of France. The Palais de la Cité, the royal residence, was located at the western end of the Île de la Cité. In 1163, during the reign of Louis VII, Maurice de Sully, bishop of Paris, undertook the construction of the Notre Dame Cathedral at its eastern extremity.\n\n\nAfter the marshland between the river Seine and its slower 'dead arm' to its north was filled in from around the 10th century, Paris's cultural centre began to move to the Right Bank. ", + "\nThe Parisii, a sub-tribe of the Celtic Senones, inhabited the Paris area from around the middle of the 3rd century BC. One of the area's major north–south trade routes crossed the Seine on the Île de la Cité, which gradually became an important trading centre. The Parisii traded with many river towns (some as far away as the Iberian Peninsula) and minted their own coins.\n\n\nThe Romans conquered the Paris Basin in 52 BC and began their settlement on Paris's Left Bank. The Roman town was originally called Lutetia (more fully, Lutetia Parisiorum, \"Lutetia of the Parisii\", modern French Lutèce). It became a prosperous city with a forum, baths, temples, theatres, and an amphitheatre.\n\n\nBy the end of the Western Roman Empire, the town was known as Parisius, a Latin name that would later become Paris in French. Christianity was introduced in the middle of the 3rd century AD by Saint Denis, the first Bishop of Paris: according to legend, when he refused to renounce his faith before the Roman occupiers, he was beheaded on the hill which became known as Mons Martyrum (Latin \"Hill of Martyrs\"), later \"Montmartre\", from where he walked headless to the north of the city; the place where he fell and was buried became an important religious shrine, the Basilica of Saint-Denis, and many French kings are buried there.\n\n\nClovis the Frank, the first king of the Merovingian dynasty, made the city his capital from 508. ", + "\nDuring the Hundred Years' War, Paris was occupied by England-friendly Burgundian forces from 1418, before being occupied outright by the English when Henry V of England entered the French capital in 1420; in spite of a 1429 effort by Joan of Arc to liberate the city, it would remain under English occupation until 1436.\n\n\nIn the late 16th-century French Wars of Religion, Paris was a stronghold of the Catholic League, the organisers of 24 August 1572 St. Bartholomew's Day massacre in which thousands of French Protestants were killed. The conflicts ended when pretender to the throne Henry IV, after converting to Catholicism to gain entry to the capital, entered the city in 1594 to claim the crown of France. This king made several improvements to the capital during his reign: he completed the construction of Paris's first uncovered, sidewalk-lined bridge, the Pont Neuf, built a Louvre extension connecting it to the Tuileries Palace, and created the first Paris residential square, the Place Royale, now Place des Vosges. In spite of Henry IV's efforts to improve city circulation, the narrowness of Paris's streets was a contributing factor in his assassination near Les Halles marketplace in 1610.\n\n\nDuring the 17th century, Cardinal Richelieu, chief minister of Louis XIII, was determined to make Paris the most beautiful city in Europe. He built five new bridges, a new chapel for the College of Sorbonne, and a palace for himself, the Palais-Cardinal. ", + "Diderot and D'Alembert published their Encyclopédie in 1751, before the Montgolfier Brothers launched the first manned flight in a hot air balloon on 21 November 1783. Paris was the financial capital of continental Europe, as well the primary European centre for book publishing, fashion and the manufacture of fine furniture and luxury goods. On 22 October 1797, Paris was also the site of the first parachute jump in history, by Garnerin.\n\n\nIn the summer of 1789, Paris became the centre stage of the French Revolution. On 14 July, a mob seized the arsenal at the Invalides, acquiring thousands of guns, with which it stormed the Bastille, a principal symbol of royal authority. The first independent Paris Commune, or city council, met in the Hôtel de Ville and elected a Mayor, the astronomer Jean Sylvain Bailly, on 15 July.\n\n\nLouis XVI and the royal family were brought to Paris and incarcerated in the Tuileries Palace. In 1793, as the revolution turned increasingly radical, the king, queen and mayor were beheaded by guillotine in the Reign of Terror, along with more than 16,000 others throughout France. The property of the aristocracy and the church was nationalised, and the city's churches were closed, sold or demolished. A succession of revolutionary factions ruled Paris until 9 November 1799 (coup d'état du 18 brumaire), when Napoleon Bonaparte seized power as First Consul.\n\n\n", + "After the marshland between the river Seine and its slower 'dead arm' to its north was filled in from around the 10th century, Paris's cultural centre began to move to the Right Bank. In 1137, a new city marketplace (today's Les Halles) replaced the two smaller ones on the Île de la Cité and Place de Grève (Place de l'Hôtel de Ville). The latter location housed the headquarters of Paris's river trade corporation, an organisation that later became, unofficially (although formally in later years), Paris's first municipal government.\n\n\nIn the late 12th century, Philip Augustus extended the Louvre fortress to defend the city against river invasions from the west, gave the city its first walls between 1190 and 1215, rebuilt its bridges to either side of its central island, and paved its main thoroughfares. In 1190, he transformed Paris's former cathedral school into a student-teacher corporation that would become the University of Paris and would draw students from all of Europe.\n\n\nWith 200,000 inhabitants in 1328, Paris, then already the capital of France, was the most populous city of Europe. By comparison, London in 1300 had 80,000 inhabitants. By the early fourteenth century, so much filth had collected inside urban Europe that French and Italian cities were naming streets after human waste. In medieval Paris, several street names were inspired by merde, the French word for \"shit\".\n\n\n" + ], + "expected_by_offset": [ + "\nThe Parisii, a sub-tribe of the Celtic Senones, inhabited the Paris area from around the middle of the 3rd century BC. One of the area's major north–south trade routes crossed the Seine on the Île de la Cité, which gradually became an important trading centre. The Parisii traded with many river towns (some as far away as the Iberian Peninsula) and minted their own coins.\n\n\nThe Romans conquered the Paris Basin in 52 BC and began their settlement on Paris's Left Bank. The Roman town was originally called Lutetia (more fully, Lutetia Parisiorum, \"Lutetia of the Parisii\", modern French Lutèce). It became a prosperous city with a forum, baths, temples, theatres, and an amphitheatre.\n\n\nBy the end of the Western Roman Empire, the town was known as Parisius, a Latin name that would later become Paris in French. Christianity was introduced in the middle of the 3rd century AD by Saint Denis, the first Bishop of Paris: according to legend, when he refused to renounce his faith before the Roman occupiers, he was beheaded on the hill which became known as Mons Martyrum (Latin \"Hill of Martyrs\"), later \"Montmartre\", from where he walked headless to the north of the city; the place where he fell and was buried became an important religious shrine, the Basilica of Saint-Denis, and many French kings are buried there.\n\n\nClovis the Frank, the first king of the Merovingian dynasty, made the city his capital from 508. ", + "Clovis the Frank, the first king of the Merovingian dynasty, made the city his capital from 508. As the Frankish domination of Gaul began, there was a gradual immigration by the Franks to Paris and the Parisian Francien dialects were born. Fortification of the Île de la Cité failed to avert sacking by Vikings in 845, but Paris's strategic importance—with its bridges preventing ships from passing—was established by successful defence in the Siege of Paris (885–886), for which the then Count of Paris (comte de Paris), Odo of France, was elected king of West Francia. From the Capetian dynasty that began with the 987 election of Hugh Capet, Count of Paris and Duke of the Franks (duc des Francs), as king of a unified West Francia, Paris gradually became the largest and most prosperous city in France.\n\n\nHigh and Late Middle Ages to Louis XIV\n\nBy the end of the 12th century, Paris had become the political, economic, religious, and cultural capital of France. The Palais de la Cité, the royal residence, was located at the western end of the Île de la Cité. In 1163, during the reign of Louis VII, Maurice de Sully, bishop of Paris, undertook the construction of the Notre Dame Cathedral at its eastern extremity.\n\n\nAfter the marshland between the river Seine and its slower 'dead arm' to its north was filled in from around the 10th century, Paris's cultural centre began to move to the Right Bank. ", + "After the marshland between the river Seine and its slower 'dead arm' to its north was filled in from around the 10th century, Paris's cultural centre began to move to the Right Bank. In 1137, a new city marketplace (today's Les Halles) replaced the two smaller ones on the Île de la Cité and Place de Grève (Place de l'Hôtel de Ville). The latter location housed the headquarters of Paris's river trade corporation, an organisation that later became, unofficially (although formally in later years), Paris's first municipal government.\n\n\nIn the late 12th century, Philip Augustus extended the Louvre fortress to defend the city against river invasions from the west, gave the city its first walls between 1190 and 1215, rebuilt its bridges to either side of its central island, and paved its main thoroughfares. In 1190, he transformed Paris's former cathedral school into a student-teacher corporation that would become the University of Paris and would draw students from all of Europe.\n\n\nWith 200,000 inhabitants in 1328, Paris, then already the capital of France, was the most populous city of Europe. By comparison, London in 1300 had 80,000 inhabitants. By the early fourteenth century, so much filth had collected inside urban Europe that French and Italian cities were naming streets after human waste. In medieval Paris, several street names were inspired by merde, the French word for \"shit\".\n\n\n", + "\nDuring the Hundred Years' War, Paris was occupied by England-friendly Burgundian forces from 1418, before being occupied outright by the English when Henry V of England entered the French capital in 1420; in spite of a 1429 effort by Joan of Arc to liberate the city, it would remain under English occupation until 1436.\n\n\nIn the late 16th-century French Wars of Religion, Paris was a stronghold of the Catholic League, the organisers of 24 August 1572 St. Bartholomew's Day massacre in which thousands of French Protestants were killed. The conflicts ended when pretender to the throne Henry IV, after converting to Catholicism to gain entry to the capital, entered the city in 1594 to claim the crown of France. This king made several improvements to the capital during his reign: he completed the construction of Paris's first uncovered, sidewalk-lined bridge, the Pont Neuf, built a Louvre extension connecting it to the Tuileries Palace, and created the first Paris residential square, the Place Royale, now Place des Vosges. In spite of Henry IV's efforts to improve city circulation, the narrowness of Paris's streets was a contributing factor in his assassination near Les Halles marketplace in 1610.\n\n\nDuring the 17th century, Cardinal Richelieu, chief minister of Louis XIII, was determined to make Paris the most beautiful city in Europe. He built five new bridges, a new chapel for the College of Sorbonne, and a palace for himself, the Palais-Cardinal. ", + "Diderot and D'Alembert published their Encyclopédie in 1751, before the Montgolfier Brothers launched the first manned flight in a hot air balloon on 21 November 1783. Paris was the financial capital of continental Europe, as well the primary European centre for book publishing, fashion and the manufacture of fine furniture and luxury goods. On 22 October 1797, Paris was also the site of the first parachute jump in history, by Garnerin.\n\n\nIn the summer of 1789, Paris became the centre stage of the French Revolution. On 14 July, a mob seized the arsenal at the Invalides, acquiring thousands of guns, with which it stormed the Bastille, a principal symbol of royal authority. The first independent Paris Commune, or city council, met in the Hôtel de Ville and elected a Mayor, the astronomer Jean Sylvain Bailly, on 15 July.\n\n\nLouis XVI and the royal family were brought to Paris and incarcerated in the Tuileries Palace. In 1793, as the revolution turned increasingly radical, the king, queen and mayor were beheaded by guillotine in the Reign of Terror, along with more than 16,000 others throughout France. The property of the aristocracy and the church was nationalised, and the city's churches were closed, sold or demolished. A succession of revolutionary factions ruled Paris until 9 November 1799 (coup d'état du 18 brumaire), when Napoleon Bonaparte seized power as First Consul.\n\n\n" + ] + } +} \ No newline at end of file diff --git a/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/sample-doc.json.gz b/x-pack/plugin/inference/src/test/resources/org/elasticsearch/xpack/inference/highlight/sample-doc.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..881524e46e18639782402236d4aca01ffcfaf02b GIT binary patch literal 388098 zcmV(^K-Iq=iwFqnhEZn#19M?*aBO8QWN%|GYIARH0PMZXjx1M}9XRi=@Md&@q!9PQ z-jC9VWHK{Z%1pA8`H-L#3Ks4b5ytE8=4s}BgHb{O)jiXsC)I-<)=X7E_e@_<>@TS= z>9x+;$IRT9T~;ZDMngbD1?6R&?PkZ$b3fKz`#=5yD4@&>pw#^rz(?|FYZR?;pK&KYEkk$IW=|wqx52+vkt&gZ!c2wfnyu+w-|QK8=ef zq3zs^@BZ@qdg2Q{8c)qIwtv|k9@^awvLBh;Ur&$PK3|US_=tZ$Jz{f7SkZmH*ja zul!(W?R!W0@%@T8hNpk%zyBA>9-H@TqKiJoQS@hj{ZqdFd2<^2+zR3%K!22-W`(7Ct3WDe>eW|AOG&qeR{v~(_npk>@UOl`OQ`L@H{qsc^Udsi|73I z$jeDbm7M@$d+c-$PC@A|?007JmW$L`3%yWi}()4o46TRdjl9}exnKW;C3JQ@FZ zY{qAF{NDBa3EKcu)gNBpZa#0f&1H1&=`fu8w&BX&4E??hoo@y-@OR_3^UI9?L1}Cx zd~&mEPUmTou);AE-TQWfm$dD9v+tgsxwG!^6MWaUIkcO1$NmMQ=!g9aTL!~A|LE8DhZkFI{o`YYHM{@d zvWt`Zq>S&&Iz#84n+L4pm6hk-&^%!0pznVcI&=M`u+aG>%g$iRcbjH|8NqDsH$VHy zH!b zJY!V3npYR{+s$X&Z9ljvKlff>%azgL7hfIEE#C9QIj8t}^Lwts7wo*v7hLmh&ffF4 z-(H6IEmoeb@UW6!Y=+%daO2JYI(A3wve2)sJ!$WswA?T@@6$=`j?n4!y*{a3+lg<* zpOd#)ROVS#-e0$Orv|O@+oCM$tV(WL4==tkP2#esZW`x_bV7NW)QJc^x^vJAM0F~y zjlZelw^?0e=)sMEKXsizS`~S6`v7+WTXS01S$@;bVYcSDOv~G5w-J4NSrnPP{_f7- zt!OXuERN;z{s{LHX_>`&EKhj<;F=SBTovEG;AwaAtx;LV`R(J~&Enc)ep_+-@YYtY z6N&NyLwNfLUQ50?iJ~H|Z>N9nM?Z?=0_{3_UCY%Iy{)nD>O9J10J%hiL>kw(E9mZy z=c*Sd>O`M#YWtJx1O<{JE=6Lx*}PDmWkqq@)2n;d%VkBB$roWO`qrdK>hyLsVY66k z9_R8~p1-ac^B7f8a=Vo{8&f-$^X=2Qt$lkLsSsqKN?#vlb2W; zqHLI=Z6Z0&6ZOM+IN37Z}}{(v$&MSdVm#Sy@g7EQ}T8=a7#qK zJxQQ5-M;5LJ2A2(PU<+l{f;X4t$6P3d>_1)YVFu)m5kKr^hT3N@R_l8-xgJdTL|aL z?PRszpS-!@gwndo^>}WNPy254D<`e$h{x9LHoJ8Dvk4R!?AuYmKK9*A%OuXr{B{TN z(DGl3(?6+Xm+qV6)1`T`USdhqv0Qn7W{uRK>)XK`x~+WxR(`6w;h{NMbB=+MQ>1&E zZqqcybsfpE?e|XWFVr+iYmBwbs5K^T5N;dWUDv*E_CW@Bn8?tq^zCt3VcAP3g7s`d zT$1_if<3g*vQGV;C$Pyu14*SnFl#Rde`=)|6O7BP2vxqnYL7vt$e_3CxO8?tr%{T7 zO_q+cWt~6*-2m^o?e7jcoqq)`lQOF0N@|+W8m%0y!7BAKMI9wj9appiYfrEP%G(e1 zo((UNLqmyG59%++U3f{3fj~#)8W?z&coCQnRW6+!db_I<+9zoy{h$fdcQzU+&4k-E zmO%fp<`@UGT+c6Uvk#ww`>v>D#+;qf?@gq{r--WW{U=@mHy%d*c0a-Rw&tSBS2ome z8F3ASN5mD(kz1MWxPxJCH!o}+bhfztnlmP*8xJOeL5<~epwC!yQdPB#$HQP|T|~Pn zfoZEZ*1;>n-F7s&sLn-W$JD&D1rXys85W7?u-J92o#-(X`#KRL(rKa7W3VxGev-)M zFD`2=WzmN3?ow;!J5o`Wy%}nsQbsUDWu3wN^UY9|acRji#iv?3zEiX^=hm`zoVBSK zlC)mz{do;#{B|hs?h=)F;-;$xd$@cUUXF zJUM!*TDJYJHIXbSBh~rc$c`q0qova8p3Y`30QI!2Wg(wVp|wiXZuXXmy`d;ztI4SD zyT>-PCwU}y_+#JitUb%hOs=G-eg9yKG%BFi)UwvS@g8KbRf|&2tQ{`!bGXDKY`Y@K z#D`r4Z4)Y?sOHW4CNyK>X`@J3WQ4&`;LkgW-(>No13I}A> zOnMC>S;daXsGraKcCbYhCq)YVN_jAT;f*^PPW^D+JYOC*Znf?<53if^OTX#p@;JIvgCV$XpxpT9n=hVm z)3xJf*KU5+Tri=V&wQH?9h{%LVfW_YGIn&-+t28s_dx{z?yngFvY)-(d<|=?b&)P8 z8PWaZ{r`B}jF$&=cr(6^=l0-R{OA4hhkt-VfA9M|ZgOaTjL!KG$Eksyb$r4S*zEA_ zuWvVBokQGbS0Fzz>*{ zX=Z&l{0CYPZu(_^-21uqZtUyMo}U|hPP=V8E(-qr+r{&LGp(G@UK)(W`2`k_#a^Zb z=p%LvF){FGwv}Ie!}hs5x7+h&z~X|TyTe`j4y%jr`9*(uKeY3^tGD$$B=5VC8`lQJ zh0u=t+U+~{qIYe3av>qS@YF-pMPA2gklwi8x9q{$dUPE=a&fj_UdA8(0Tcc4*TcyD zV*mZiW;pa`Sdds1$M)>LB#5?~J)|2>R>$yXyE*nDz=o%H!)DuKvp7f7=;L}W{{7YA zL{krcW%#V&esGb#{&4Dc{blooyC9}6{3>k7-N&Ed3x0_UdpOR`zwUS4aoMkLx-HgF z|1as#!9C0U7+VC(1=Gtw<bPn%?fw zmF}_Q=<{^??XSN4_KUB5fN=A3jN7?+!gDY^8+Y{LIH=?QhIk%bCM!xCck)LrpoSY$ z)bzVMOtEiH@>t?wwsoCAT0_wnU*Fx`-U4cQnyFv(?hY%;w`XM>i3cwnv|a={V=cO? ziI_x@6r!gsP68*Dq@_lB{Ml;lMN*2paS?l@9hr)rG&1nlw;cBI>fI3rFm7_1uS zoT8v{os_kNIo;Os?NI13wqkn>JKqd7Os%i!8iqhx7O5Orp0 z8P9&O#w1VGb?eSFFA&9bEFaiS=YSK6D%vEvw(J>D;Zu3U6v~BST$SQ=byuNphXEUj z*YwHV<|e?Hl{kzp4^Ba;>nLH2@ut~MRxeUj+{jfeoC0@uIGcTYUe<*;J?1E5MQd8R zH{H%&CduMTJb+jFmzSe+v=lF{zuQeBMQ3imbuyUC3YKQ2282^_yadjySPmRhX1qjJ zamdsX_N}mjVU&Hq8%3uX_U`Ry-QBgJ9R{JgJ}_G5C&$MZvHBR%JjKjlLX~hC@&Rd#fmF`8|3`P>>ga$pO>k_7PNXeS2Q620w7bSQs?V zc?#Xyr579!p_6$Qya~u4xib7sM-gjTgunhe^dsTspAx3|fVRig2)Z7O-VVE|E#$yjeT! zS=r(vl`r3(;Ik7^JFe{kFBw-+l*rk(uxT0jr6Xu{8JYyvpg6I^h73=^nII8nXD9l0 z+$Oc`g`sce=29Ie%jgti$H@wIDVP;bD66qw)O!Mb&myMuN2pK#aS__j2!JTWm9Pwh zWKQLjuW4u`d4Zyg@<=?elR;!cxL`G`WcH#Lq3&ABuuQ>}EW@7CU3CTc$k1EyY0P*J zu0$;Q+s|eA!P<)|Qar}Fe`$k2!d)ffwGZF}tYw@aa$3Xkd))?4Q5M(Gv}XS}}ph`FPdz zvEW!r@RW4)^bG6kWXQRsrjuEWgQaIbpaD z1ouiI!!`6FVqG$3y=MMc7aitKV&dJf$2}NCU_)gRFqxemUMhz+m&q~O9p4l36++t; zv})<>zT5Aoz;&`VrtcSM%Or*JU8wGD5DboZo@w;7HN`qfVRTj+q*{OglN3g};!AP! zzi&feyH0C40nW|$oi)R7jbvu&cC_|1$x2zkSCJ5EA6X&+J*N?b;lwbcm4uSp|QhA&7nYb!T==Qf=o)IAsh?7`jw3zGDG}*BWqzkeMN5KJFPY6Wk zutCIAGP~hSD$Da)fU*;Ih?kO}i60x-oOZ7<=9p$;VbAe$hP5);KF9V9GdYN0$-rom zz(_ZQ>kF4Nw6j9``NWvg9Hx!Sq!gK5J9baU8DbFSS*f9=L$EUA2zO=>k=pt0a+wW|G^6nto zIu6$%WvJeDtRSfVIG%?EY%+polgk|YvuA!*S>pClGp&w8J2Z50mY9VWl`yJnaqM|5 zZi-3gbsnoFFu{8vKy;^g=YsWwOC`!8frfa9SqN+AETv5yD zFs>LJO|nm+O&MB^E2^&5TzllZ0?;p_8wbaioH;X^hz5HUXMzXb_ck!-24lbL2n8=f zLzsI^{1=@8>m!^faV2U7{beC|9A&r^)dj!349yfi#y+ZPKdA8#nE($+xp#LLYlY5T zN1FO$sV!atm!|H*85w{pI;&t^qzi#RDQ5f(`;l2)WBa~kXpSjd?asnZ~dk+DJAZq-!XK${FbTh+f%b`uUZ>ep)`B^*gN}C=$T%I0YHy4+S>V&^N zH@nv57yj^{8{!3qcJt7TJ{0esZhMs zRPT$+>7L8V?8gnW4mZwTStMlokl~S2_`uT~!{T9LE=cV^<|plu-Lc?zj|1PQnI4Z9 z&YF*<5=gN5*}xC}cHbN}=k|DHkRA^`Hf}BT{#^r8ZTF4K3jK9YKM^DQIK-B3u&DQp zN#K!nKMf}XUYl<(%*EJzy?;HNx~+{@T&I;2TNw>omAs=v3p)1kxEwDp5S5!>yBS-w zEVh^LP=T&dDRr;ILyOICzr;hQHT1=EgN|XdJ$EPmmf_fTzntJNvCm&Oe{tE{3jGZR zYo6cDL+4yEn}I9&u_H>5ImCV#KZB8U;c6KDi^exU{HOgZ2aPd?^f&s`Hx6~3+e_|t%!Aj}*; zKU4ShQEdD91oX(~4CKvc{zN-Yat`8>6#3iDpOl^o)hQOQ$s({>(B&s?@+W@`c!{`* z#H+Xn^QP3E*+`<#Q8A?mRA8`1U-Dv*^QFDrJsUl}Om5af+gs$cjvHx~04zEu&+)lS!1ZM$qhlvmkxSkh3iK05gj`dxgx>a|k6CNnPL& z(~OV+U@4NkPzT=-z9EjvT7m&y!}bDEoyaKdW(&{><>Um=RF7Z`c^T&JO0ubUF69N+ zqBy2`Q%pbvhWG`LHw7BeT4a$t0rz)l?HDJyYN+4X7o>$l*N7j~97}l`7n&i7uY2mI zfPGR59D`PgHDeT4G?XRyAcoZpnI8*Wh!w-$VAT4IFD8mH)uC`fux2JA!}&6G&H?nROi*oses`PDeT0vrQ%!r z&e>{CBE{Z{#pOHp+pYQ8Q?deR(uLo!P!#ei7QEcuoyC1ioc)S_xwLkAr zJ}eU?0zr6vCQp(HF&AFO?K%OvjBr%TkkOyu+l#8!OeCME7T}I4J}Z$3)Ky+HVb5hL zK5%PS083@NjR$7R_-!oaUG zIcv@^M(wc2>6)b)`q}lDhy9fH4MR_(OOO2kOD)8|E8gRxmpd=9NhGOaBbJdtnu~=p zVLkWf5XX;P0SESmTqJ}=DmH04fkGZO401hZpP=MS0x1rB0dxc`4{XpCMujO5Ne4_4X+A|4a-VUC z#)T+Ahqj%N&~ZeJaUwo3bm85kj74>oB$a$Goy68&;Dk|ox;cjSyslEgSiSTij9uZ1 z&lP(M+ay>IFfprIVJR~XF~UdS9isgZsO+aTD(PgAGbH#}LhKjDWT{!!L-P>YQ>EE@ zd2WX}He4!DBColpA(4K~Z;hD>{T2j$@15C4~I{29;1`_0QUuGWovJ9>W_ zTI+h-xyU?|h7Rn^BW^#>soU<)-F7qd{dx1R_I%_Uj15n(_Tx_-Rjm_6)46vo%`iUS zZ^jpBfggXmAl&8cGPCVs9$gx*`|->GydcE8UyeGyYV)%ON5__&wGZvwzpn<2_h(P- z6oOq2p%1;{o@QQ2m`En5V}) z=J3!CXHt3Njr_)b6F5hvX$>*!kH-_{f8*%^!xtWpgyh=B@Eg3{J8dg`@z?tu=AU!9 z#mwNF-DCKrSIdh;;Ji#R=)U{o|NXzb-F!t51m4q(?G?L#n+L4WW6w-?&%kZ(^Q#xj z$Ze+0;r0CN^kG}&zn-|nZKb($iU10~amgg$3FPL(0t4dDlViXuOieACK5^9NZW;S0 zL2v0XPT%!|jrzv(XSm-y_2=m)zDqxMKzdYS{VRZMPGxlFtt zGiCOva4FjGgjg^CHStvY2_m z`F1Ges^H=6tp#8+nBy8H4ly#cXK|(kF6P-}!g)l(B5A(7^UX;C`$bd3`m;sfp}Z

pSn5bXnhp zM#Rhvi5*Hi6%)X+kr|s%+PI+oGjPs0i>6)}s39iUSJcKSghGL0g;Uu~LYGiZcBb3) z{^F@NT{d{81b&kjlj+!HrtFSfHNH8oG(u;U1=gBwk?&KYN#KT}C7U*9_o`*_&k0j>LeUU>3{> zCx^Cf5_6O@c+61u+DbG5nB7XXJb_>!5z(?217Z<~shGc?iqm9L$txHZwb(k{5R`2q zdXq{V8B+kIiiqiz6Lb;PtjGu{Yx2sOTt(y%(!BSJZ!T+CIGXM_M*_-<;Bv`qr{-bp zupQNwy}Ju+=FF?9b8%6a*N;^Dm1aXZmCp-g*!3&CtdlDmn<`UC#0ZtE3*nHGsGv&X zw@(bD2e&{;ie;rSn|;2JEk-BMlW48FeK|* zI(kKR#Yn9Z=H1;rTSsw}X`&=A?m&K>CTY6nexH)HT|T$udK@RWSI$tej^?W+sO7L^ z6*ATqo_~2!$H4<`1w~A;Y%3b3AJW5*I?Az~~Y4hT+W~4|c>4S@t96goXiTg!!u6*HkOqm2n zQyr7^AxycuF$wilHJlix%aw@yyJMhABrQmli;lF6I;2^e)Dok43g9iW{V9O*iX}Uv zcT=z4U^)e%Letjn?wXLXi~C*AC@czV&Jqm(;5ZNQ;*{=H*?vx!oAPwEipaY_!%ez< zxwp-imy%j`g$OzP)&e}J#PQ6G;9_E!KDQql@4V4ildIbY|m{L)pN0$P4x-I~c zkgHp(YaE*06ph1&D)eD*8_%VQ=SpL-WP47EJXOvb;g=`pWho{LRjrxi*s$g!vwI+GEvj#SP0QZWucs={Vv4wO5UJaWN3i`ikVutq6aKE zfJvc~$~C{AbYuvS68`oX4r^{5g=(l(Xci2hae-q`PrYlNVdDEV6CI0eAIN-SmZ{NF zFf&PT5v{OCJodwO3e=VH+WBo6#yBk$oVK8sLOYAM$cfX3pX1z6=3gEPg9*T*QgVUG zf98cU;x)xLKk*BvNd>h^g0I8**>I8!7h=n*;byJS$`UO};ZyqU2{FSenwA{0nq$JK z6<&w6Z&y_8yhfV8`5efcBbZgv!&4wwuCVImcJjfPkeuxZr$yzMp~FnurLM1=JhbQ8 z3W>9G<$3`Eq;*>?BYaf~%TN^k+u|=#6_dFDtxYGpq^${W71-X0_fe~CISsP zRZrGw=-*J7@tQ)CalISg zL=x<4Wix9Jr!GWPq0_PI&6*`XeNaSZx>9lb(1P3M@_i6uNtd$z)1rqywzP;6jhBZv zK15_rcnfEI+a6(tjQ1hLWKkFI(sxxpeXPJyz=Yu8Za+OH9d?T_;@i#FGtVx6Jyq0n zd5r(`H-9_!7gi}8m>G#@p4)M=WkPhI&2_}6mNLgela4I5VOh!_{vm|vV1Ml{xEx&}r5&zD_VZ@!T;wB97=7jA z9@?koXixKwijalre)2Ixmyc(c4C_D2NA%ct7Y_A4H#aO6`v~FCPh6GO$IG68DuU@87nLX*hRXQ#2N-7zTVk8+ zEz`I;yPRVel09yk!|A!RzuNG9{_6VCLcqH8dxn$I5&zX+x-Dlc+F=ByII!>jg>UfD zUmv}4;q#aNvfo*d8XwTl(bw>iZ#Q2&lO_RE?mnh*QDLjzvk0MoU>AJ$6UOOWCho>% z;@&SJb)R=+8<>Lhf4*s!G1Yww^K5tT%Z|F#+sBK=2+=cw{2Z|`7AOb!p?&n#cf(TX zx7*8!{=9qs_{C8P6ZMQI*p7OJGH9FVwovRZnPvV)t{KwT4&{2P%Oms zm?e`7jxNLtQx?EmaToZ2TalztDc9U4mg)$-Wrg8xj+*5OON@OmAi}DtWmQV=9*z9Q z6idcU}zZJOURimE7e2q?hcKSQ`2y-l#8{qlC4ZSlFwTfjD<-OscE-}%+<^Y zR$fr=sAS-f;MNa^YngGYA&$Gd=Pm>WN$MxFZv|%wiC#~)vA6TLGQ3kTI)*}<3H#z9 zRHKb)rLPdAnJ0?@y<{br9EdM~Od zve!m@dqJi-aS?@k$jji^NHn7VKvlvEP^n+x<~vO3y@{4vv=CFxxUG^mcOf;HC~GYr zvkbGpYH z74E)W2aB*!#jJiYt*@hnDak}7NtxP?wU_xy)=z*VqdC%C38xHOd&XK#ACAIPgdyr5 z397PEUab~Jz)4n2@z({3-`gNWH4A^;4noM;t%%>1vu0YY7AvgssMZ79j38WNiS%!d z`Vw2B8Ml6-tR!(kswQ!F*@EyQ8LVwdQJiCv#&K(bBlI7!}{98zT()hqR* zpGf891xU}IulVryp=2RB*kbW{ElkQX$rFvEO*sFO0he6jn2Q8!Tpg8?tuDi4{Ki!M z76+&EIf;Uy%2GVGGn*!}t*TPs5P4#)I26PXfu>XHEfnVbh$Hi)4v^jtS#iXb- zfp!W*r_kGeoX4DGwOUCK%2OM<7^MQr z7PXTgN3n()$5WtrgxZm;cqOM=iA)f#)gL_D%_fy5r4nTNbH?J-C24!*ZkZFJinprC%5sGJj|e=UiZ6h-1hG$?I~lvyP^oR4tAWeY>a$) z6KaPuDw}Ja-@pixMOC#@uVqTcfteUFGv)S5T-G{CtHM#^-p)^O2$&GJrU)%z5h-r? zSSv-(LB_On=d@nn#(9$#HH)zR2wDjeS*DZ{S0P!h=|U3ZppMxDretJNUB%W2?0SCZ z%Fn}SF;QA>%oCP!yS>a=*Lj39K<0JA6EN3Zu`=fK2qF>gW?30@Y#U}F!^SJtfGlBO zrD??!|J&IQRIe!uUn{e%^Wd65MA)S$B9=1^Loe2L>4*mY6Z8t$>7LP7WdKhC7gTYOs`eb9s1W$0=lL#s+C(BPNGTRT1{jJ z*wT+HVh%(NoioNux;_L^aD@)>jMCy|d!4F@6Qlfr8e}Gs)HtBm%F`I^pnw`utU%s} z00Dtsu#$E%LU;4+c@33H`uGy^j?vDL^6kQyklZl?^m^Yf;DfQFXIi17K7ubUKms|M?BoW6oNrFT*U*G{?EtV?{r7(T#6 zV3_MJ4G^oSh%;6bSy4z0+(7ne?;zUnW zH6g1^!1l0ELaDoG=DE+P`AXIieLdFBt98ZDn<*E_OBX5H#saqCV~F`set`ARhNpbAN z90T(K*Yd<<4{bG%)B}g@XaaHOvO~*7`@bB)^Z$DM`Lo5tnR~m#h2S}9T`DhbObeX7k1ClWOpt431g|Ecgq#_nT%OyfZaaYow(URt z&HwIjD$KI?`u4av4URa3PxLI{Zn#eFaD3tqNC_}sMttpf*>2l$e8h0=+DCF`1c=DD zUHjC|d<>sdRsK(Z^S4!1e0uLV;5`n*Ay|m__<|PVLL7(G*?qkGZ~W%#UGHmQdop>~ z7Y-x(5f%hNGB(T&hIC57Z%AY`FA$G}@qi;Y1{A8R(m&6}|L2#dXaCFl+rDvTe{y*@ z!C!3VKiOTjj^Nz&fL$gi$GX;#P2bhCUUu@{o+ywmKGq7tdE5qzw%_86I$Klil-7+d zQ~DQ7rvIgf=zPYB#e(@of5D6X?N`5B#}~!P%t{e@_1x?NM#!O5SZ|FQ-u)&hQ1>$i z#p6r-nS>1zHrzC00g1M69M8i7)#h0CX$fpBHz7=Ks04t?az)n-0=4M%?FMoA`%tmd z`^|2q&v2Am{D~if-+lGf{pObq+PCh<-+E-&!!J!{K92C>uPU$Wxs-%o~=(EnqQ@WM*+B%oF7+uN(ZX$4YzO2Dp+dW?_)K zKdGuZk;^Mr`V8AB8RuNd4Y4%#A|_5(Ld_FoCU@CZA4IJ{O-oh){WI=@SmAV~!s^bd zzB{%?@#L-ondBRd*^ba8>ECZe9FW7Xt&u% zmS|51?X8w+dEzktqe#rSWo>o5AeQfY-v!0qWwS@h!iMRcikE9G*Jm87t`+J^qy{@b z;3G?fPnCFb7f1le@2hlWJj8r+!U9E_q4A?{I=saf+TueD)jFyzWuKbo;mUR%Y>ETcSi%8ErR44w9b zs)o9){-0wX9H>}CyTQP}Mi2$bC$b@1BGFAv3XT8k}V&#d4y{bYR^yZcHhcFsxRYV&8!m;((Pv`4-zCFb7oW zN>z~sH;7}&uM)+=_+#Ho6jdnzF?Xj1bP0A_B&V)X+k+xHrSKWaz@!994 z!cMO%eCLD~X-LkX6E)Nb^$pB0FA6A$!6Htd*IAs2A}~i!^E%E~ybp8CFlWNK`g+Z# zF_DP0HtHD~DJY#aEgrxk+nlL$86=@p(6$UcTN}l#w zj%Go23C**vMgOX@&{#!!|2y_=FTfl)4UDzH=a$DcR8u7ynu@&Saan5p-Z^g1Y7rXx zUt%b87Q@#3wwaV20`wo=damZc8+y1#2|CB_vbw-ms`qh$Fl1m=aWg(Q*p4-p5zZCN zN!N$98$w{7+l z$|b|p$~RqkVza0$i%M4U74Ry`W6abC4$19;Ip0B>#ax!0_nP~Wh`_H%Mgw`meeuf{WoJoll z&EZI{=sK5(=@b)9S@BfP;Jtjy5ZoxJ@$>p3?u8^#RkuFhc?*uc6EKRjg?*Hf;Q$jQ}-((47 zVjY51O|b<8GQm**(^J;AIgw2UO34a1#VhW%8d3~QBv_{<9Et$ONF7yi9IzY-dFJFa zw)?|G7=4zv>@M#DYe}7^8h>lQZ^8rOtW@aQGT{IcxgsNLzTboxyc zXZz4SO{SZ}S_!Gx1jLUq4zPYTG!BKhJv2{&Grx`re3Tp8ek19Jg^s?eer*HGIyhta)q@h_-c zGUA|xJ@f7^w`N%7#me-BYmeB#GFnMtoYPifQYHvC7S`@?o+bD(D*&nhUyiAGr1Fr5 zcK;MA#l@+!LDi@U?;u7}{1b8PZ?>4^Rx7=1x|wYzp|YT5HD;JsjI*~?*Y=bTMv45o%gb1_c94aZ<8Tt2&tk;~8qUSVJhp(8Jp66YnK{r&qcG~;Yi=WTl? z0r;LgHzwztj=BOZZa-k z&{d#tPKc$19Gkk7=d2M(u&S-ZA%^ySA9#is2FX)ZOIy{^h8Q|6YTaE-H+pbkl5(?Te3|`vQNSdf^jC&gM8q#?8-3A2NQr z8CuuqJ92*MH{*dV!-kXqA0laSpZ$${DE|7h;fKFRk3O0Hu>bhyKm3<-OR&V|cl>J4 z;!aKpj%;veZJzr};DlKg?wg|1=&EtII4YTKKb(4R<1-Gw@rCC`|15`gm>55rL(9;% zqr>VPiJ!c6!dypr(s(|P2rq6W^IZHH59^TZW+B+A;t+g}|GyKojv z!85MU)8;c*=IJsrFiwE5|HxnTm-i%TF)NyL=yavsf-(PzRBaq(qvm%*%KscTQ)q89ZlC*pPtuj1%thx5 z=^Edh+h)toET($N=(50R{PyT2C4&6$U=ClXg(#N(blFcBmifJ#&!++wBbV_n9DS94 zANoKrN+sQKzj9C(S19DMzhIx>%NFZy#IbPMKJ$>lpZOTrc?KN9g5umdU$*tb4{E5o zfMNRzA4Bkvcl&$H*0FKrP+-QjEykpA$D*qn&2K%oZq`GWTzHi<9x?ed?9(x?R zLHB_JRADXbjLF41LH1aOnEC^~L+3wyWK;B~ATnr8B*IGf%yZ4R$4p>ap*()?L*ulH zG9{x6cnvZ=D(CwcNHX#`&6V9gAQG4x6)FBcoHb5{`AM0;_-E>Rmp3G?q$h(s)MOrz z`+B0ma5z|rD_*gTWGRa{$p=ix0EnQAn(A#U#Y?bg*R4;KtHxTHF(>BFWorctY7#4o z;v32zz;KD=40b>klqk{Uvr{)3;DfxU>d>EPN??g-eaV2jh*=_6^x6<@ zV>>omMNchbwpu z;oTa-hZ1Pbuv3-&(c3IFY(D|xc6n!5KJhpE9E0zxH>CVw^d7U=JMyEtI0fzNI7{B zBJ~|n7g#`BV0dX*bk;GwP+TUY+mOi%vCj&F0@GKH&~86fyCD0@3h_f|PqZBP6i#Fj z-b7NkV2n|%ozNc@PD(A-J0%psCd3+2JJzrZWf|3xhWcjB&r31Nq!B^b_z%IkQIa`S zX~X6?a!G!E#aV>NWr@$w+Fa_V3>(Rj z6~~!1=VhU_Y-SV(6XatVq@XvNNJ6L2Z7*jC1vy(1B@1!Ns|i56mQ*iyHHMlYI2v_5 zEZ7*($wUvfnUo&tCr)08j$Oe~SX5k;2(uc!2=|9P;|knRX1T%|uiW8zQE1^F8YsTi zA%kVnT!puDsxzARjaOK6QfN*4U1zWb7J@2kg=a1T)_IyUe0tl5Db_&}pEX^@d$BQ3 zwE#arz`vfjW&nE$XoHxJ5A%f)MWRWwu@!~Qp>QeFz)gYLoZK_2&srQM$HtE%0fWr} zOuoKWo}gE5KG^PQ6tBG0&QlNnucuj&F6{2b!4zx8DjeD*az3K@x#~j_5vR zuBvQ~=ZiHaYxT+LYM+#@w8GI>%g$H=m$BL~J+fpE&8wv~LNjDC_&-YmlrpHHD37_u z2Z1D+@{ zp`F!`G-kG}TnN`;)08#XRW}aR94P?%OYM2O0|G3mCd%srrIdmUbab)UVQ@Fcz%P@f zv4-96?hHkM&(M;SFtM!>Un;>RuQx8>G-b(J6l>k=0~U$+Dv3c|A=(lG@3qR_a0#`9 zFybqA?Gn#a)ODGP*~AEPh^aw@dD*J7SZkG{3lcM56k>NGf~1+ejKL;O@;WV40qndp z*hJ7>b;bnl%OtB|Sl@QVdyY+@SV?%F5RD{TinU<8g(^%0rcWs@0JEwmBveG4uQ+AT z7prpzfuztVrA52ENLp3Um&LePu-drOS9cVytV<{*#bj{t5kzK7CN5H}@I2DKcL9<` z(m7Guufwv5TL4;d&bh!sU!gG;GHgqf5sTEWVwd+<=q}nixzdZUn$90DO+Z!=dZoDZ zySu=6O9IptsOKU~oWwa)1{s`1Ex!#1WRHSTt?-^!(5dP;_^J^oJE# zMDv5`GzI~v6|pGohMj`Wn64n+7LhL#E9y#P0W1SwT!qKQza76o#V z!xPs`N>N%03swgiocgzG0jBGICS1X0t?@EZ@Mp5R7`kDt8`j5Vy+p|z*p<|jP>YJ~ zO5w9a?#88y7|CMF0-moRNzDo`{{+Q@_L(S_Ye~8kF`G*c;4RC?h~lwVw%@A49=iNgbW?w_?^r zR-U3Om;CIs?Z?_m4}IEo4gawaU@#Ju}v!f(*f0 zlanQ8Y?(%RK~g?twhZsdaRaZg&QHOXX;I`cbfs(s;&)u01`&r@Sy882vJv^yDowOp z)>-ArM;dsss;@H*BA1`Ef(>r5P8YmmDNEXwqJRZ-UX|kpl)KB_l8etuinNZ!ZTixT zQ!_tGTyf{isk3gd3Mlk9<=65-3tZ-o7DDd~dj;>5SPi?z@=e&DX~EJsDy=zZQD*V? zUomJ#F)5)HlD+MR<0Jy58cK<5>&tN32XeJIEuC$CJ25j{mW7LR&4!*CVIx+diwPQ7 zNc`87Qv1owr1Eo2C2=bMj8>j0Upv!9km3FeDVSu2w%f$85;xzT-)=sqX@Udx(7t>;Zhq1b204->%+WRl zN+uX}7|+o|xLohP8TvDC|6^dSdgr3?9=y2ikB{gHNpBpW&&|(iXq{Vh2zbo%Gd`iU zdHd>j_lBCnk4a@^CBVX0b>o5mNys0k6~C~j*o59+O?F^IB0VMrxBI5^n85A8`d(Kg zM4gc`Ig+q|$?6W|a&&aD>nV87J0j7DCkU^15Y=CkWC4FU_Ae8Pb=>@}AD;}$IBlLVkN8Tzf!w&+ zIQkjccEi@f`~DrrX5f4f;JFy2fTHs&9iK>w&x`xW$7YLric`zc4?K;2hf#rF;B~L? zyj%-K9NE@e9AI&|ga&@mu=X4V;@4RB*mv#aezSe<+D8^Q?3krGvqt^fuSgrkKJHqt zr+6Z^1I&CI+6m(L&^G8lInlz(3>e3sw>G(tPzoFt*-$GJH2JwXwR;EmbGw{otfx`? zwjUli{Z=<{P*F^#Vm!38M#!n41rya4^dYUxgJAy-{3)lhCY8cih6 zR4uu?yFB{llvP@VS|gO`!Z+l~9UO3%Xs-w@Tj7NX?W`LiHr)aomt;{Ci)OZf<|UP; zHZO9CTm>^&{4&%bUPJ-d7^3!k*M|ygDJ$mQ{+?HzZ;q5w$)5)wdA_G~C8uj^i`a|g zr6e)U{vyW-RjP%JJHcAhT%*J9M z1?wjC8e2$fayhb_01AvUrPqy}+tjtjT80D~w~6Mk~y8OVh2w$*F7wP?vpc4Esl7 z2iIlV;w(ciK8!?mNI6(rYY0N>U`~X!vm%}SySvH9!H8*^YjR+-K!MRSy~5xz)#!q1 z9x0M$!6R0(u7zfoJJr_&cnOGWRrrY~b(!Rk6O7;~GGvtEQI{mzS(b)l8GX0#-(l`K}&HYV8Uo?J9V zA?4nzGm*5^=rN8!AG%n|vfLO>R|V^ORmcP|WIw_pM`nkWL`Abpya@KB;*d_0 zUE_r!;vIgB^QNx&v9hV0B5e|G#PlAF4K>Bsb++Q3ozTsBl`)M%+(=7L9(0LVwUNok zn={jUt(>H@wM>N6V_GBe3R#sivs_dD7twe+6jxYEyxQs|h$k01x%bwVWaPU_vb%s0h&$kooE^C`K0FD$pVAP_dER_8xKsnI;3P;##i9y2_)tGgysJEcU zz!Ax0T3LQKHHfmRb_p!E3!TP>31pl!nUZB0ldQz;pf#Ij_<2Si2&J!_D49x@LS0S7 z2%&yQjH;-G2|k$>1%tT)8=7ihk+&p~^W4^F_znzdmdkFuJcTl5I5x;3H!ZczywIOFKDY^Mw(2_eJ!5Ng&Uu6%Sh8JCTD{eBG-|U zQ7s^XDei_;yt&>16C_GCCze1r9x#H`xFUB-Sfxz9c<*y4G908>1d7EP0uF-^eWmW5 z5ziS@$4kZG7lGC*VqH&JrdRA@jAp3reN(74tzjF*g8FlwBNIqgG$4;iHh~xioD4}OSnIdNq9nCASGapNsu;U-&l*(B(MAXs{ z`EUt0l$`%M5 zzZ8I463H1D=T&GX26OFlwDzpjI%(WS-M-&6=KDp>ACqP5q7 zy)omG^WVp)bW3vH4oiX@#} z;U*F>p;l}9F1UMeJC*fHjh1C?;3P}ej;Jf7F6)xj$r^ZILG_jLGL*>HBKra-I-}9z zP6&m`SjAj+?gE{{VoP-?P9o>93w6nGLajMO3Tr0wpp2`4ermF-%Kp0$yfR^auo5d( zO|JsT;rrrTE*=gt1vO%@Ztk(pmy#0pujN!w^mxsu{%t-oRZwbw`=CT zCzZvWS69`Voohi9lPV(|2eKC4TPtBmE2Ogr<5wc=RsCHZZh;f8f{Cn)Rq9f)g|4v6 zF}a-dn4MY0Ads)|AYHKt5<{D>a9`e{vaz(1M6LUC^RDd!sV@`=I&idbWE;wxRV;0- zdX0|-T8Hv7c+;tI1 zFF<}dbxyx@0Yb&te=cA1ggv*NWzr;8SYJ@K1Lsx3nr&+9yJ{!6vAh%!;Hzmzp^#e2 z3_}GiTmpc})mqk*y9&)DLlc+gbKlP!B!a!B@s07+ZqE%B2^)Xb;Ywt2Ug7>C?14i4 z-Y#evBjwJJ)nl6^^wP*XS|{CnEM*hJAqmkI?0K1vN2~3`S)|^o8cNET}#q!z6V9 zKvHJLD3#=Q6fd8V)T24iD>3<=TvRn6f|#wJX?NT2yFdz7lu0h;{4&_=#d3OU=4{HyVGj%OL0&=q2+_;6s@fHfID$43RSLF0wL{kj zHn5UZ9aXiE_?KgT{=bM)dup*M5dSgb><#OaF_e$x=%6#%J4R*B*@N$=}8IQ|UPa?<{1#wHLnK{Dur$2Nxgl5GSj$<+{Lu9-GJ3_@U0Y z6s-cEgU*mdX(3Butr;rl-ac`s`7eNS6%a4|W#7KXtdAdHulm{Gphuria&I?ZI~FdU zb`Cc(0dYJMDJ;Mr{{G3MRsQD-3w*!%l_w8hqFa847CwA4uk$aSo5N{*^TU7Izm7lrLwm!W=4XMG;_%cT{u9202%eW`{Ez2F>j%!- zo9!M7!p0IlE-vF^s$kb#_@}4Mw_HPSKEuIB3P0nx3#H5g58Cun`({FoxWD$V4sW#! z(|+o)>#g{L>jiOXZcutRqeJ`T7~E!fmLKaG_~zc^IsS;(V>LCX$D^UFm;u?3J^p7q z?qRA@${PPh4T^iR(5$!FLV4FSPfQk3ZWJB7PWHW?N9yQ+R^n7)08E3s?^P$Kk!W=!-;(R`uCjyeyR+1j3{4o*oh+a z`>ipd8xL3oRZ^zriVX=uC6=IazC+gt&6>q?ch?*2h}aKV<#SxtagkLkEZQ?aKwZKJ zTB+S&0UBpPE1eAMLL25MuQIK)vqW`ZuP17z`D4)wu%w5idf%iaFj2PUVcWgyDieV&VmYC6d+D;(B@XCQyZ56!RnPA1J5YtK`~ z4f$hxQQ?m@W5i!FY4~1Zo+VYD%U3*4Kc=%f5?qhJenNde zrYDOca_#kIWjYOrJ~q~f(@SRK?#{R-UASF|Ad4!1#nM&S^jsDzFKP!pS2fa_vCzbE zxTxJnN;g4aeBX7}J$#$yHS=_{x953TDpn1LQ*7;VS!*S|*+H6<7BZKOX*Q+FWJ#hI zxIjCpikEWGH{SQ*C2+))N+z~-*0o6#nMS`C#T3z=DnZGvz3EDV{VAg|qX(F;r>E(X z1TTpb4T1)5kr%ebQj7`F1F=v1WXl zsqaOgd8`nzUYD&a$t@!1kyC>f*f>@JjI~b3RXjH(2`$>kpEh<4a^tfMbfy|p3-F!nqwSoYfOsP4y9=2Xjqa?GBlwF zlUa~9UU807_{ReVna{gdQf*_8f_5F@n7(ZfB|w-74s%M5*$@cLA}E;3FkZ5u47!L@ViW^0tr~wj}lJ85I(Z7Fht%WA(KKa}h$83gs zN{-PR6k-aCGgXeRsR&QX`o1e`djZ}m99ycKwnJmhShuoSXKZPTyC71s;Gy46pe7s$ zoG!f&0zR5dWU1PWb5JKG^fQ&!@L~~HN6jEJxj_7;X_;4%9MMbk4}*{^73~%TVJv*8 z2FcxBSS?9ak#cMO_%#ty$%~I+$);m+ain*byD$3JiHxjvOe>;?gfM_f6~y&^xa3&M zkyG~dnShA`v6jR;;i8@KU#*lxQP@jFrw;T zs1*qXOJRh8mM_WkY^6B3cPM*_3`c@QKM&3KhJR;ur$~v5U>#2-4C_>RA3U{S;30$k zjU!yJsivX@WEsRt!6oqsoQBJ(vNAD?BrOkik>0Yxh_x54PPC{iS6Ik|kC7q2;zvpx zao5_NLhqOj4L#9ow3efp_QaPJYOzjHgCN8d+1ro4fq_C3P%z3BOA?ECmtx;qz+*-EXy#{dU|v8-GzUUdkA zO7Op&11??}+UcoZb@Ms>wh2(79Ea-)qsx+DmI_;@#3G|%t9Z8RC4YsEiSx*jD9SxQ zF=j!aQug{8>kwz8-4T!BO{H~5P`gq`A+(ltqQx?<>sn&mFeYMwjN^pwh_i$VZ0es3 zs(4IKgt8qQ@HvQ(BuBzdSm4&qDtB^BdqZfd#K<&A(Oed=nK)sZn;@{4=^r?(nWg%n z09OFnMahao`-)UQW?_I@j-3#SG#PF{QR|2Hp`F=^qq-_HtqeKk7f1AmX_SiYS1SNm zXRN5Ak)m^N?Nm!;R2IyJQ1;Svg_HO0t{=lQaGqD<=n2e{XvVG*ma^c&4n_YeSRu;a zy-4$jV1<+#jey306L`Y6!AQ`U-!+y9=5Q1%Vz8OZqC*6I7LaPchZZvtamD#65*T(- zAQp?${UV5JaxohCE*ZKUPWrW+$8Bi3@1*hWtMJmm3dWH zvF6^#Wffq3xO&n`Oy5O4NVns##F+_1I5cuUl;@jiGY$ubsIV!N>vZ99(U zJR$Tg&eXMQ5kV$jSISBn=kSr#%jR5+Vc z*lW;K@rr`-dP=GKF1qrfUo$ zlteAWZCE1{$uQfQhKOlim_*8?BI~Ky>^`VMo%;)=_LuKTM)pJ+;J2Hv@fFGD>+QMs-vC=+Nk_IHJ;HJ{E~t5gPR-Cjg~Ok} zyBtnN+(a1pJ^9ss){yB~Tq@tb?%$J1?Qu-R2cU4cVA6Q#&_SBUH3rC2Upe9p$zAuqWJD+*!t9%(9c^!GG7CgypaKiH=8~i ztiYrD=>kPa>fJ3#-U8zEe)Go*%eL~3u2_ThsAW+!PdLb82YY6Cw4ocnb{J92#BbVR z;LtQg<@y;t_^sa%j_FXCw&|H8jgKT`b7qX==EI&Fe7RD;n7FE4wUlYVyyfCHh!4*y`-h7VdpIB!uY@d@W;EeD3 zsaw5Y;H8)Se?t*c==_zaccmmDKnSrYYQbt(;^8)+ z7x9E-h1L3IGbS&AwHmKM0zJJ~3>~Z%AKJoe?2DsnQArO2iyg1Jrit*>`Y1pSRYzW2 z+8X8mqpF<(kO8akt_}9#mMAD_2MGQ;F=2!z<6YeE;)#+2|-J~u#ov|h$Yy}T? z4&@zdm-Z&;nh-`qYo*!cGZR@^6f3LDpi(uf52q3UTfm%{(y1%`3jR|QTBEEKIH7$r zOv^M8t5Cg#H9bmV3@n(rAdn}y*lZy%TSv7^NxEr%Y9@-}3S(J)h*XfxE31XdqO}$< zD_1JTcA@`w)tXpY1+y9CcwOtM{bWFCA-lr>9BW&W9y{0Ui4 zS6C6l73qZ%nyhPgRlsP{10rWjyC_fr8^2g7*4K350r*hm*`IJJjN+`&zt4K0m#Q2s z)a{Ho&EEuQdaM6&>667}P?XBlSJqZBG)FgN@h&k{CDUb<9R9)jh|4Te-wK}Zy$EH( zR7edEUdPglnnV<8lTVTCqKXtWwE*DIBbBZIvnp$)KS>LxO=VMKGPN*^sf-%Q-ThPG*52Z6j= zSy)1iz%r+*&Y1P+C8{b>WZi-^F|QKMPd{I#0;l-7v`=x}yeOGEdwU1A?Rm)%7i$fq zcFS2emv~NIECKKz%uSTzC5p&hW{s1KNPhmVI|o=_R%h7?c7B0e#>&lC>I-9=w*kS1 z173;5ycKQ&kPu(iz717c<9dyYXvxBm$7P~yTBMzw1~bNiBN(^#+=Lpa5#42~6GoGDyz)N6v zYw56MQO%t6p7M=TJeqcrB97g@xAu6AG-#0$gG~{ylul_vIzxtUj8`<4cHA1whkI&; zyXvOWDa(-7D<6{EV7qE;a?M`>=5f!H5L%PGt_lEWuAQ#RbvW zKUx}++y^V+s_z1B2)cWjEBayb(RraLTj3gR-ggUjG1wrr1}WxrgOs(+GzD!$XRWabnc6R$&T0uttfOA=RMP z`}RI7FG11+)$(V4X%+>|R>J)I?(kp&c!8qQ%t2zPG-4WJxoht3_5sj`YhBLxfB)NG z{hR+_?XZ&+bT|QvQsN#ZSl+LZ}v84ogP@jTp_&NRpnO=&=tha2N<>A~Y5{DDQctm57`pA-sWw110qR0=*!p ztVu%^GFT9FFYvzU+?`J0P3@^xSbrI&L~5ADwNh9w!omS8N2l9Lw$2hJ!o;ub2pJ4+gwOFL?V;2XVNF$S) zB$cb|u+x6whSGq9FKA=~KxB9&>ZI_7&NW99b zJf0>}JYVxuw5Kt+)gofZ3pJ(pyiRNxQx-D6`+$jz*Z%9Z-g zQ_>tP;GBG*Ylo>V0LJ@x_{c(XJWkzIpp^8+KX!pl8z754OjnGT?kQkF(<)=YX07Dz zR5&#)S&3N^w}#eAhI5SLR5|i_`4dF)Vp7nUBk^fH^k)1g- zxItFFEmyD^$Nm8~tlep3ID~a6$NOy(m!mnv7;5V9vQkbI!CYXy0XasN6_m+*PE3gC zb8Y`LG^gj#vn+9F^AE#%v2Ac6M0L3k_^+wA>HtSGi(IIb=veo5xcN^W@Dv-R+=A0^ zM#Hg&20Is_2GYnS0EP_vez?zV+w@@!bjkd&eN~up0 z4}LbBM=etM>z204qHdkL9zPyH5BEh#roC4lwX0ZuZRtg8X!e4m7-&uZr58z3#T>gwhEe zh&I2mB*V>DOK9W()Ayx$-O$jUk%qW&JV=l|Es5R5VSeOz884?}KING3IwfhT%S4)4 z7(Ifvf8RN_%~0Ug!7_X$$-ld~YJI0pgw{1sY=+D|7+jo5P<>l#zzh@oY<}!}N z&fL#P9srVr*XsndG88G1u*OSAO1iB!ygz@B@&4(qF=RRIjS3`O(Y<(4@%BJOEVb4RbR!HP!n# z_nZp`ITY2I%30T!w?IfinJ%|uz4EfA77Sc;g-E0gcw%R{?>s7>R*r6;N;2n-hlEKu z1Bw!M#e(W61j1Nw?J#eV-U{6*6tN_I%m5G$71QaZ%#hZgt(>4OvNm6NgkU4F75B?6 z0HNhjGk|S|YC>I(s#i^n8`GNYRO=tW(b{3Qbs#C)bs~=o6HZeaz~6BWEecJuFhIJLo4*a*c~mF58x!s zP+QILKxF+{KqxZ4NzB?a^_BBV{5aA!IZmXtzr9`_zpN=ouIHMXv;)H&=bBkRIDIq< zzeDXp#;b6ykEO;nqsvoX_Z=kJ8 zPL}ZZWm|R}YbHp+(#jr{QFlzd3vVdv-%$1=dUI+ct~s|9>#^ocKp(&@7|QKMvx5Ls?Qi)x1et4h%=|vchf@L@K0y7ps91pw=zx0?R4ohryAWDHx^nQ0PBw zn3(1e$`7(Sxac`EXQI`W2|gn#tdoGS&am!WwAlf(Od00{;nqnkSmt?!cfr6g z7sWt0AcoTIp~TWao|<>MCH%I5Xfx#{7PJ>)lKEJdLZ+ZoW60ck?uBn;q1DtT!MJSk z%UZ@M?IRroSPwf|Dsu>}Mj1)nSZHl3+h-*0LzJvGC3J-8^G)E<4JcVMDvb*W9vT{G zi*RAMj+=^`oKr#EvVpcegJu4x!`09+aRL$@I$rh}88+FXKv>~T;1y*WR3tceXxlJ`?o2*Q5AT`ueG!+5ivbqs3&8JMnK^E7JG^oH}9vBFQjt?|Pv@nv^(Bf+W0}AB>S`NqQ{GRNA?GF6dW-zQp@s z*ZUZxQsq{dQ82))SRD`^WaCo`YJshdux&Far)nYZNlU~zfojl~;5#7SWjqDa zwpKoKekUnpRK8x&8C;qK`jbjxzMdA)Ol;VQd(W*WOVvU#Y8gwY2^eot!#$#~N4)&U z5cZ%kq+wo7Wb||#bZHEFU!6BjRxc0$_9e4S%$=bQa@eqQqIocHVL@dR%dKmsjdbZl9fu|uc?t5dte>KgZ7+Kn{m^c;I`7vJ-`F7)#AjCQx~WWqqCRgx zkNYrQao^2;#x@#O*4&Oic`pEZ7d;{UFK|K5hwy5J(riCgJ_Vty-I}~S35j579 zrY{LKnaVr^hbnQ^dp`m->T(YJX%{{LD>L15RTaf|=ZXX^K?`6=$LA!IB{n7_7O~LW zZDo_;yvTydS!^IfOy14|>ZmP;9271}NMnzY!-HGL6VezEJfcI4 z7UX>_|B8v6m*g=$C=jFWOuHHtRT${YAQRlIx*VfL8n+weP+Rm4X!Q~ z%2-Y<+C)5F+sON*Jhv8y)dbQ4bl5C3Td_Tr40*UDazy&1rcGuav^h$s*tFc99Exvh zWZE1_w-#l+O$~i<+6BtKv^tJX5k{DtkP8Y>SrnYet&$ig$}aQiiqJtvu?on;SYWh} zuP|Joh`k^s9MqQ17p;D)NmJL{wRk%vXJuLW(C_j=Ty0k^AbiA0iy7!H7J$)^o`J@D zWmQ@hX8>S3XjIFlE32IghzWP59puEOMpycL(uMRoTo*L(8_rB>?kOHNeP1anqFknD7L!!|wkr{@Qc*gkItOr+d)q=WR86{} z;pF3bdpAaTZ?S=f6G_yB4aY!z3LKZ0{?|;hiSAy*Es`_IDv+fdjsAEJZ`RTO)GTYU z2{iP`Qi%V{8E|lw2Mc(;x!PIEX<0+ij9%dMxDTC=AjK~Y*R2*TsYsm}@}5yZ4hHgl zvFO&oRp(P&K52fALBTh6zDKWf1l34--IFR_o1yOl7xi@#P^HV5Ezk)GNhz~O$!O~B5U@H;ZTsNCj*_&IMQ zpcL($bl!T%n3{-pEc-QeegjJ6EyO(rvnUr4Q_x;S4S6$V0a+06!iNw+^m(abZO~MiJ<{-C^ZXg;Cwz zQC#+a`SY*G?HPzn+;oudM$0U2$0GpO+W_D5fAOt{|BV+!b_sZNA=bNKMl2qtO<cjD2^CI`V-vg~ZPo7dnqKKA)X8-W&GwhQu#Xn8c+dsSt1d;Nr zn8t5MeZmE0B|1A(CFqy%(jIn4K+@dZjX&-%2OxFU9ys99c1IUS9si@dkVH2_@sHcJ z^DZ{UmvQ`BNhj~7(Y0p}q2`8Qio9ghCB{_4>!4tY%cF_>HrC%MM%e7FJ9@!N6h{%hy{>$lT(w?2HL zOTZdc0=K0+&_W6PvlmQf>1W}65bk>iCxxWEQwF!!5)T4vE#Ht|c{8{*fRn=-(?e8+ z#a)shD12{j6ZGZxqkR3v_v@9!Ejv8M?fQNrU-#NS|1YQQVJ**iLV2h_^X2@S*9Td{ zkD%;4MZVgbwd}aAs^z4*$Mcg++I8{y`394RD~>mikzbu~hyFJ_r-H-4KH!7Kw=c`~ zd+;KkJZ9OGtl)kfc2hpr-ROuhPgoi!aEi*x|99|AipSGrWZ2x-{>rId14_Z1els;= z+kq^Z$8oRZOxS;9PPTbfK&_8izGEAd{qgItR{ zPGP!co~n7{4n`%zcRe-+P}0tFvGwX|TyFy(WP`gPgSBfe1(viMm;vfpPC*@pVk@P( zMrCM)h&q!8K2E2=71>lhYib0q#GBN>^-*`&jFvT!2&GWsaSRVAVMI@<>j3Kl+_g8X zbE4Y44h?I$^(wL(4ichw?FZXY)vz1DY{YdQ1ecwz08QZopr5sLe zf)MopIZDQE#IraU)l4)XQ=#RZ0nNmWYuY<^!&?|raY`*Zb)JG29pzBM+^nfW=sgwD#tbCJ1D&7pL zr_9>0l&Wq_Sy94mAz0qkRm+J}w>v8)waziN$r|*dmHQm&d14*Ag9IIGZX}qEPD1F0 zv;4E`>#SOF3mPW;S#TrCmr$~Rd5ZFDyUuXKu>6#c+KyRae9Y(#TEJ&np(o9WgPNYnb|wSdT&9gz8&I)*()L+fBCf6G}( zW$$Yt!cx<&HEW*(pVP1ib%I%IhQ6x|5!zfrK0%n2p;S63Koxq0On1A7ukQ`8Y0##F z2-6SCXULu9i}+OS04d9CJi!t2xMy(vZv^7QiOarzVexV8KyJ?1oyLD2s zzovn<2m#;|)-E}$Ov|ar1hUyQrGeIZ-oaNBQ*sAXA_)xyAN5uz?J^Cz^=Y#~*sa4> z9+h^_6kH<*ff=YUIn$dpC|58T!AD@;0^TYu;RuasB+6;&jsK)$UByYb@VX;58kKCC z>C0*e`1Iw6GQ4@mpuIf6_vU?IGflCQSod-sFR1E1u;EJx980P%u@A}#&!?1z9at{uOI-9!R8RorcEkn5 zplan)HMDvj<1o;^w7)KagL3{ezEyM0-lA!%vZpO@k3^(5sB7`b`R)5%Oij#C@-knD zgv$p%FtwaSdQ}{+IVSh7Ho!i?4Nb>zz*|*_NEz&tK#Hv_P(Gd0K9vgm6D&DGSf8q_ zKmtolX~+(G0}uu>QAbF~242enKsR$EwI?EF*@Qk4Gp;X8Pa35vIiXGv!Tfuf^(=k3O zgC`Ytq2mw=@vS$Iv$dob;PTu$E=wk(We{IY(|08$uNGZ!V8lVrkv@|IUu_!%F0`Lx zA#BzWUkP&BOn*~^adz!sa+%W6Y zVdp-fpsQY0WyyhY%IOZK-GPy>&xdsYa9CGbW>JHkYcz@2B60H2kV0PT{t&n?ty39( zk7FFLm_R*pUM!(Omc0^r&O9B~@nOAJD+$jSNDmOB9|wq|C5+MxS+Hd*;R@x~WgBdC zyaqs?6Gn=mbD&b%B^TIK98OFwFz;EQAR8E182Yfm@!7Y`R4bdwE|~~ZB7t(bSQmIL zDC`Uwm`~T~I7D*D{Z0u92e@j~KH($g0Q-c>HRIr?00DuU)pO6TPoYPS$!_S-1xIss zGbO-_vO`9P^MShqn$!dG6Tr+23O7Se*p*PbV%pb8pxIU+PNyAV32czXgrr9X$rw;h zWs7_&dH*O-hVe3!6(Ism+2kY!Aqe<11^sfU-7;muP1I1YhFbQ0T3P4Kj90D1Uu7vF z0ExpmBJmuJ@&(yv1!k1@BO0qR^BJK+?#Hd$DRYmyiKLuE&EyNm@hMOrmae5M?#Zu>s~aqv@UqgII|Drwhz4 zn&lzvK+D2q+SCNwOQy!P*_`n(1uCp|W9BvX|5z0U|o0%S3e1pEptXBA$@dK6l1`az@n7&hksstB3YXbz))PO>? z$V-x+F^g+DWS@8m8TP6192BP84zGmV^VqX&0qbUAjV)VB#NTzT@2fO+w2N3$F@epuoM=%0Q#CvpH zhqquwn>KfW;qUISbL?o_=@*4;3IODkjt){`LgSz0KlhvQ#c|FZAIHtc@l8Ig?*YAw z3hAnbyj~TD-Nx(k9^|XTs3Vzl*8o#MtiKG<)WFI&|9F7n`DggdWnKd&>RYH3dg8zc zR&=`5Sc@L>_s`RQkC0w@6U7PG_<^sTpILx}@3q{|{p7Ts9T6Qa{UVOoQMof6x(d@4 zk%`}cwsE_C_p|&r@Rv}K_S53UK+hg7c{4$RdRnbU2Uqm*k(i1{4_aHjsSg6WQa-&e zd_Kt&zLkIRYcM<(zuvv9Ci#cmaoRs%4nGs8L+<~} zr?L2l-D&jzpy;)OfV~vox*g(Lo@adbe;teO*4yjihxP4tdMQ4e#MDcx|1fTzpbb)d zF>c=dpGzR*cax`30qH-y{=rU*bdhyFT4J=1|!kEgTC3j4_sx9@45 z9>p;F1f-H-Iw`&4QwNE~KYx0MJkQptXP@AZ14g-XaLq%27;5CF3u%2wjRRWkm%HNg zhqcVl_Tjqt=ZD?7_;R}$*X}pIdG|AdW|^0FKi`h>4yNtBZ2vkhPK3PpH4v-fNE(Gm z`UU5Zr>LwyI4;Y;ZW*AS00-Vc<@$D7ZAM@|A=ajdV)vME*%>&>lP z#UuOha@SjrNPP@I+B@X57*rT)3a zTouOS?x3JuP=tkTK{hvewzWSc3VD-z{M4uC(hhm8~)UuNA))Tal6L#emEgb z@bBP*v<|*b(Zrt=x(6<@n!q9$_ru8Tk>~d~9u!6eLEwQ>h_`YQyWtio7Rpfqow(6) zaSO-`)0R5ubl2cfp&V%tzXo$7`2s&;7xUvl3{tw>oe_Rqyd(y3;j?(U{E&?lvu7~y0D^E`eg2lWd3h;Wh;e~j0g`-3b zY=#QZ9g{ka0E0`Bw;Jp?hD+O0x%~!*oF6Z3%L2_m$rc`lEkR*Y#9>)xl1g!3dZnrtz0sS9&(Hb` z5g0Jk1dF-Ioeec)*Lh9WihZoQy6tFDV*A!{z}%RkY6d1kk+EmOyO`RBGo2CXd33H# zu*mRJj^X4`n-*!!%(tZ^mR*@?LdPlMhoR_Az+~A$e2msNT?XxvC^Ga_1zB+pbre4( zy>fhi%QsSstzS~(Q^Op-f_4FfdBc(3gavvKXMk~Kf z$pDSL)S~WsZZ3XWPp9puP{b058v^WT*q|o1V(OMifZT#Bhc=6=D@{^E9m10IVFK@H zhNk8e=pvEO_^tl~c^L7#T27wKth-Goi<1sY#KeBMkw0hrzB@dEL6uPycFJ?Xz;<*@ z-h^j#*%F#Ub)eM*9V_&%WWx%X(FVw62Fgr_$iyjsZdimXrNFFlP}6RZVDlSvDR2jm z-+O~a#pFo3rlwVbv3Pzd$o z9WXMKG<^uAgj%-co_Us1_EGWd4eOZQ>0n*eOxUyF5UQ~NTh@l%Yyu9eZXK22hsw6H zf7=Wjlusaeiv~vwlUi+`c0prX7ASK!s(HYR%Lc7rmoH@#C^f{z#};+@V$!-=h0=|^)zOMgOTh2bP7fsTwg-d zkQub#_ympYvSHb)V~}(Pw`)y)N+PE0f&$ih*QZU`5nJ3T?WvnJ&g&3rRnRRZ;EbG> zVYKEl;M=u|`#NIrioS^@1`=l-qc^;6U+_z+251+qb-Cnmut0^I_SB&KTmvV`KQb5Q zWJ|UKhaz&UM}AI-8M8QM!ty9vM8#knUbWrxO4IY(2bYSCIIkviW* zUd=M2GC4hsVHC|yoJB$E(OZBLi5ZC7K(~tRZJ^I1Vl!<&un1tt!+IOU4H<?$lM!cj=?bJ*-i4hZWu7OEOr z?VquIDaS0Ged4Lkz=o0rG5?Mq%)<|044?@pFlOu276?=j zIe4t>MdFfYQaVxVS=nJhn!owuSLVo?URNb1z&y+ypgt#FReZy<`FF98Bw=K4J!p4LzCaiN~r6Y9~q zXWjrrjJcM80R8LAp3$h9ni5BOFnj{ZvZ{0(Y)9M~7m>RX=yp9bfX(Cp&c?_$m}%Xg z<+(xa#7VrA&Dw2PmiFAonT4~IiQ)wHq9MQ{D5daYbG|!0k9T(gLRgAVO&{tEyG4$y zrD%!RA;CFEr_g0DC*+hM+)S$^ZRT`Lla$m_^(aR&Dd8I>2j3$3r*{k_k0B@UFh*_B zvPXjpLpv`~ay^7_XiXoF5S$ApRn;=E;U>l-P@QdqHXiHNayT`q{Ywq(sJr&)OwK9r8Xs_qtxhkut>($%sYy-5UBdwIV?0ar~jr~y2>6V>|=!uma(hTxx@;U@IK@E^3Wv%(mZk3>S zfqSl6iqVgcXMG>2Nf}lcnUee{^llo~pjvOA#u+3gXHh1rw+`?}Bn|1{gdBOq2n}7| zF^=~3ynVY?7`T|2jEeNkCJ5;;1_jGzwZ7ku^V7605@0|6ZdUCOH@ogD?zs*jg&N5r zT(eVh7;m&cRs!VK^c=$exPu_6J~(<{GQviqy3{fb_#Jbuxb%dY04T|9KF`m}Gz!7- z%qQ`*J_Hrut`%Q`!_rG+URT+S^0hC!-K_(X*nj|nSGMnT(R~7tyO!ln7TuTdKr%iK zTKs1waoJ%T!=<=XT*08Sl99be44LYfw{pI){Mm6E#q1?)Hqf%-o7Ul~ADC)65q_7w zugNt!1Ul0i+BM{CpF)}2kvqz9!6DRs$<0~yOxhp>?y1^ zaqR1!D_cIcR1@(}8s5`WmsPeOS&{7Ixg7K3sI3Knd<-q;0%hIqcgHYq;vUpBH^}*K zNgo@OO#^R=i1+t1rW7eIu9<-B^ALK048RQy*Pb3Gah!JhSOS8!cneKA27tR}!Z;D8 zK~jgHE3)Hn0)eQUUJ{bYL?kAEJy7$%5=7G1einxk?iko|!8vUv9|o0p>lM#={WUE$)poqX5D7XV>Jk~v(-an6m!!DZ0yi{pNp zz;zgunt%E3znrFxmkE~pSV9gZZZOA$zOXeBfvE^T0Kq(pRG<3OjeN?1Z1c{sGS0oQ zzW^>ZzJxZg_Z@bxQ>)^uZ@w!|+e>kd7=r!YY>ulG>O^cX(9&y7d4KCO2f zaC9OR!Q^@gN15}}GeC=;@-Siy=bzXe9Hdaee)3E*JMsgyDSV2^Kb@aV59pPpyIxNI zZw~k9hw%3+XN@ef;yA4i@BZ^?Dn9n|%FtdQ<#x-aCgkLSIwj)n`S1JK$>2Rycam^9@$Lr!#ee>A!RE zYd=H9^yKD5p3V{5LW%x(-kiKxyVtIt#<#C!R^}ZD0UP$XNQ>!iy{wYY^5hVnNGf}M za1^%OpMMvqR6m8Z+0EvqSUn(Q33D`_&hxDL4{&t*fQJevACZR;UpsQz#_>}=;CDJ8 zsaKGE00j6ju2O1JPhzV2xH2U#vX{gkJA$BGzR02a`%s>auYnw=+fk8kZ4aa+OV0Q& z*O|CR2f`(&iMMD<36+S=M#Z=r*V;yc0NBLccs+nBh{Ip_$q69mL9pf8vXAhs^ytnfI{s6Bn)%qJJyxbwU^g~ zqjc-Rj~fScp@j0;dk&BZcYFqyaqR_WR02JQCd6V4(3r+>YrjmC5Q-P(i zY?!i7ac#h3cA!Y-Y4CiS;!jr08NWaX!pOD^OAV#5LHv9P9;u!a#l*p$NQ(s&?P0u> z_>>k)5Re_Z0WumHo`7`_8}Dl-H(Ma<2833e!&OqmrV>v$*Yz_+-b4U#=%bV~PDjQ6 zt*D@Nj@zWSLbO8>$b&2e{f$cA1mo3y^gEe1vG`~hDJ}k!A1{&dBUr23QN4i~Dh7f_ zzDxi$Gb~gQ!Z=uom^tGQ%@gHR>`{8)wb)u6Ho4*?I9DTSV+TQPM!F~XHMAjD%)gRk zjC%<*Dkd59$Edfc!TF#0`u(V_hJlM8Gh##*gG4h7#=Afu_K;YjtmzVp(@SK;;J;MA zz3XMCVI~cnQaavdwA8#qt_Id*+K?4t@0c4~7E4cSJsogphE?Vx3a68$-EauFpxg!S z6Krp++^&9r+LR`z9eUQVwL@*VqBZmsBRbIZRZLzXfxODTTU3;|);NQ~y3%Q+(DG3~ z(c8%J042QK0|#^|U>^jYgD(}vORh~N(3ZB7xuOkY7skpA2oi3qA8%XMl376W(BYa9 zot0?{e2`_?WO~~hpMr0jy5}u10W-Bg&CuG9eyQ4+D#M4E01aAfiPResdPf}IR89wi zD!2-V?yFE$RR}v=J|+VSppNVI7w*FQZl& zw8Wa;u-fDze)DGHi3~Ntww5yBt-L)Z(42wD4Y+i8V~qv!rfGq4xfuSV4~e`*4SH@` zmJ1k6Q?+v3(>a#}4&KZTfH7vO@e*za3tKYGB(@dEt;jx3FFplf0ERLUEMb<^sZs4` zqT&?W1yoR`Ac#oNk9Q5^b~3%xIt=M`mav{13s6g}Sld+&)lSF3!$S{oS(~d{q~|%= z9ZPfQ!frW39YvjMdHX&04$Np_M>5DBaat*_WjdLm+J}GM09A0BESE3R@fw8TlBRo4TtyV3VT|P|5P_kxJ`9(9gF!`7o2&iw7}OJhur*9Tp|SrDK;Gs(qI0w*Fqd1w{V{M;}`#WFbk&PR{MJ zIthZqj!VBu^QLW^uHjC%&3%+<1NmUdofc`6j~oJRMiqGhTdSheIz+(nrdtbZtYaS7`bsb-X8x`iC}<#rY-lWHVkkELZW<0VUpZyT09Uf}Rr5MgH8iW8~Q zQar4tXHfA{4Z0o>%Vu(@%Jb%W)Clo&8}>6`pNCBQvaizx6W|ya?k7tns#{lx_$d~i zn-0b6tus2NGq3|~as#7&7d3c!(wf6A`LiI6EXQ6=*b+GYe%ubdWf`3HaihJNCD=Zh z73?VHw zc$d4{+ekl!yC6R>!Ig_2A-gQSmkWBT0s0dyCE+6OTRawc;Fq_tySrPjB60$PGZa&V z#89v?Lz(fRWHS98Y(U1!Ntd((7wtuL&)xXCoy6PeD!Eugp&FhX-eY9@lTVGX(Tfsz zFa-4l-MLPIw`qY0;Z42NQ7iiwH8LzP!eH;NEu*6sUCyTNTZzml0T#NW3}logcDuYt zEE+^mk?ag56H%NZ3%qN`&gj=&;>N1VwjD3S2*ft>R8Ydr2Iql14m{L@;nQeL_#x$? zP*kFFNbK7(rG(Cz_GodPpTcHmE)<0n&vNmek?w2`{6HtzB_&5SmeC2slzizgb9vJ1#5Iw6sT0rIgqa%c5d~L8xN6K zpNhluh-1?I{BA2|bWh^2A7D%v&MH@Op!{hS2Udz=n?J8(H?pH^ry=KgVrx z7}uNU@uk?xoa_#7{e=Ja%ik6s|MuO_$7wu6z!j1Ph5YlqTTY*(b19bwL~rA9BaTSV zX|evo3%9=e`LNzU;6G)RK%f(s!tcq_S|3)Mz$%D|@|X2jQ@n?JF&;(p?d}-iS9|&2 z5^fYRM2ZNMuhw@b$kOi5H(&{T9=9j|{F@2lv!0&~C^Olt&yN9V;%5FA#rNTI#n;nL zeD7oNz1+k|$jEI<&t;je$Qmme65&&H#p!|N5yLViVpRHwDw}?_UnGAa9H3wY;3Q%(+W$-Fte%m3;cA7fci` zU4={k;&N;GSOS=EXJPuz#UIb(y7)})ee<$J^*KSMR8sfl{VKZpUUnd=J4~ZI<#Drn zmOpn+vH!dv2>l{Ad>>F|{?l?uqp*VA9wHZTe0*S8E3|6m2tsLw&M;s;#o0i0 zhdi+bj>g&yk~(^l>m^*t5eE>i{qi z-;{j!xquJCvmw*Uadjnon2t+)pDA+2+8q!$ea98ZtE;23+Zi-#5}GL@Kb+8N#<{yW zDC|r;r82`^G0DLh7}M&8dmJ`yw{cF?fK)Ust& zSfFFtIst5>L+G z@+MFy$}3y?@lqB-4&SgH0}2TEh>TOLZvtg4%Jc^JL`@)eNT2oIA1Z1By0c8%Slas8 z4|PRY#-K0kL|ik1K#i}l^fJm4KTaDWaKgz8W7yb!IpKXI`gyk5Y?d6d8L3Q zNN-fEe6&au_r|r3@C#9f3M>*%pG`1X|_X5OL(MsOa&YCR#H6?SSXjBI3><$BwQ~GWnt-HBS|^s5 zY4r2)lecNQ9DUbj7bp>0aK&&YUU6H19rrXEJ$7#d!cD`#ml-zYx}La}qtT$6$Z?$U zC?aV-GJp)zN#%?hIfmG$*emB61UMM{7^6Tn0&bA*6Y+XD4k01iP#!T+t7uTF=ZtkK z5nH3g`aZD&oiV4Nk|}Pcz>>zP2VTC~l!^|ZC2~QWKOa%TDJxr3K|ewlAcfo21i`_KyYad&Gi|(1_sfrWNL-S^Qj1QE3@D@d zdzuqGYFM=;;S_JiNN3iR%mlos!i&>YLDBI8sl-|?GPJZnru&z@;)NWPk+!K=>8l~p z&Rg|N#OTt8%=^4hqn9tX>6%PAQMH|GgOV3vc@jl)px|>p{5kMBO1RnPKI!91X^+sS z%!FBg^hm|3A7qv&(JZm%?kSRT%t1A!41kfRoz5wuySndGXG7t&ElYze|>s*Dqw zYN*Qj)s+OHS`_%Ix^9^BC&E}OG`jTkIo+V0Hlm+^TC2!s_pU+(?bD7bDq)*b8;Rms zn#LzC)dGAMjIT~L+t8<4vux;`;ZrMca#Ge@T}1^Z;dzL`#Kn36iR0=wv=V%d&zs0D zCP5>~bS=y564=T;(;?7k$R)$#p)h^HLReHzb?{iw8`R>fL{qkAPNs!ER;9JT%WXC7 zGp~%zQ1q?0591BQ6(iC{;t39}maJXko90L_kAaY@0zb(6!8-lW*^FxMxL=1pa*!l6 z8OrWZ#j<@1z&~S3F63h^5bP{I_7o093qP5YBeA{$l}N+R8xrb%Tn@8_bv04AyFZNb znpZljD34}1l4E;;m5p;?KElWW004|(@!(jswhdxHhGQ;iXDnE$4SmZOO$^rdb(d+T z7y=BYMbe3TPohc}5^@833C@p5^i@l+XcL#GoEHHI128tvz09qA=nnzJQZ`lDQMq3H zccqUr1G>X#zwpU-H9;AU{5TRGg@8SiAGprb0K0;~e8(E5=k0p6-UsA|cq)cCSQ0!m zqBL=Fs88w~*@K2w);@iJL{>D^!!TrMrCfPNO=O$PF?*c1t978B8Y)Nyux}7~Jpj%T z*D}9VMv+U%MUfAi3K~OWu4CL>L>!mmu7mF>_bR=C{DjT0RwFb~1Bjzh%gSA3fokOC z)eJL=dYaqgtyR z2Xgr5u=_tv!NhIUoZZ~cdqG*sv6CStOH1k{x{8>?`T`GCN*K(PZHZzvXrVgBnRtIj zEx=I2jH-v-OBA&P72b;?gdg+3b-QjriQK_og1Tmy`~>Pm6$_c&I;1vyOp2{ zq7QgnYXd0AlQQhDAy*2L>n5XV8nGZf`uDhTB2nc=rgPv^=XSaY>1WeH#)GCS!Z#s5 zS@KSQb%i>yw`hUzrJn1SHM8-yy1M!)m1>nSP7Ohs@^Voj65^q6Liewx`j)9 zvpYNlEmtSu*mTV5HI2vUVxm$5X+z4(?PU{;F@+HmIVqX9FKx9FquMoi=c2V>A^urS z?sKC^ud)P7ma)m`-vo{M_f}qiIXQ^JVR{{4E$if>y{qn8a0%nzkK1CpyW1TcM&dBt zo}ptm9w0&n0N8Z_A{V3}Fyia_5H?%id~^-IkrxRXgP_^L0Tpi6tC!UV1QvYnP6y9% zP0KGndKE<<1BcQ-aw-5`j8J6o#rf$DB8c0!!Q_Vt`hK4FV_SH9$R1wp*0Jk=k7NuG zk?-Tb2P=u((7h-6*^K@&QG*T_M6a^v9g=x*j398n=6b!Kvl;yG-C8~indl$n=XO86 z$VJ7)dC*Dv+C8~a)1JKqM^(_v7Oo4fWuI$j_S+BaQ51t{PJ zsX_3}Y>Tq^^|(EchZoFGtIMfW0-?_r3@3rICHRpk_sY26A3W{{c!v#Gc)(F~Ie7}~ z!Q>$`4Si|i`$G`jf_n$a~nxFjf*CYR>;E)Pw_{i) zpE`2FZSg%AbdT~bWm%1D_m!PasQtrfF>daB;ZD(USlMy!Pok* zdv=1AvXXAcgG06j<|KumK6((^wot$z%(b1-{iSQMcZ=pbH1vSNNk8l(_BAMLUdM}_ z!o6R|hml$N)epb=<~kBlRgqE2|L~8FAIyDMrw5sjstHs0*F#d|C|jdl%t99~BSt@y z*MRfy($&3>K#3l#>zvPzSQQDdEmL?duoi7Uw5|k`8tGnR^{Z z?-mKU_-l*DX_$QWI0ZBel$uIn5=`r?CFC(0PFc^DmD1&#xs~by`BOCwoX^Y0@CS~w z0LeG2|HQc4{}%KD$t%3^EEhn}h}TFN!^5P)i)C9wo%+)4oWVmSN`FkxIC<4b@kw)P zB{Mil97e-z4=J`A>^0oU6%YbWOt3PPFbK!;D+Sl|4A?#fIYW5@LnRv=s(#qkJ*$`} zTd22k?KSUfCv)Q+yYEYozyLu7)*au7N8rcX+O`yE&ft<3Y{BoB!a!p`>sv3{{|8WD zQu04W=V(H2xr~+)k$|?g?B&N9fM@<;1)ezLkuMqo5n zb321tFEwMu7)Xs8SwD6IYV)Zw-_RQ;Jb+o4s(kTmC_`|+yo z7;fv_a6c}qvf&i!64KVnpRs9}yD-1KV?fG-$&#jclSk9Kteh184&|jtGnWa0T5J z2Hr%~yjn8@eG2vry8s72Rw^y1*mM7cXFXKI9c>9yS6zh1&sP+*Yyr2OHeQ(kMOZdrT`w{1eTSfzK zk!*1T?=69uyOGPY3=Ltol*qI3|UQS1p(e> z;ha=Bt*$L#>@;oOkx>(91?E`f6$MoFP(-3#BtfjyRSg|FrmadKDcCD3-bqrCbI>3S z9PIkEsE*6@^Cpl|Pl2I`CH%I^q*p_vhM^fg&7l_Z9pcOFu{L3F;h*# z)>O;~>Niid0EUCy`UR^8AW$r>5^)hSEN(yTdQn9PdZif_+<3d*NJ5mJfk!iQh$8^X zR5h01$*KZGo%U&;iwzdCruqBwX)|wvdQD3oNAzgbs6yH!$GYOjOEAdL`RVhR-a?$7 zfp^g8LdBPeGlllSMc+y#{=0Z56wqFv5rA|v9Mn4B;26P2ot&lQ!Yqos;zxBYV_1;X z_TxQ-?ir?m40AAO3==nM4pmJ$5aLo*X|C=<$D+;XfXmC)@vi3RI<4M;1%hYx5Qxtp z1xRyxOb8JzIL#P7BU5!+i`7$L@Tn?%BbP65+^RZV)AdzG#~X`wHsF*d8sq|XPv;~( z>8OlcvKIV*Ej7W#;)>3B4dEd=o-_eqYv@wbuPgv=R*o;5L3booX7OuUH1=GWBw%YR z6VE+R-p>x;G)(X8V?XYK zmQmRcbK8(B;tMo%k}nXOAQ@~}Ob*mN=WT=`$^P6i(fIf@ZM7~B z>|)RLa+G_||7;&1nJ#FI)w39^PmRm+rZM?NG8&$30OsFhb*LoQ~@O(&sBoe{iByNF7t z9`Lj!Js|QUPdJHcJS_@4^4u0-_gzz53 zBgtQj>Sz+tH2gG6h@3$>%Wy({#te8fh(!0Sw7r?O5jr+V5KY0)WzVMAfpv=l%anN0 z^d-=p6ijQG+1o&1ujSmr9@UX!08PmvM^h_ZXNi@U9Yz9kg7V&N=0*cq#APO$T8;ZL zmL6oGf=ZW`*LV!qkm!+7X*Wq`&NR@7VAA+wlIxUJ46;5-dZ@=3=BE)yt7E+HCB&^k zCk=yLqfj)G-9t~oW{N#E6}qSh(;oZL<)gAqCjI6iP-@Zu0!x13W{f!v15vh?gLqB< zN8!3kj#~~JQqe)#PU@CL7x0xIlJm*P&!J_89E5EI)5%Gik;pzM7l8xqb>jVV>-SoM z=kO~Xw0<)fLY#s@lTMsj31i5OSTvX!3ledD4-vcx5$b`D$**wZ+6hl#;q# zmQ5hW)V(VOaMtS*&1q_sVsd){G*9E@u*wV_j{$0>R<woivsu!DzhGXB&$WiY{x{|3uvwH^xuAUkW&u_aPj%hm0}&DZd-i5 zai}ne2Fhox(93cC=xCqTk84j$B^#0KukP9_{R6)b6gSgm_gvV&{P{2c_vead`g-y$ zK`;1+Z&Vx@|637T$HFNFdlZ*^o8U{^dhV}rqf|;B1nNg$-^gwZfSs(R@N@@I>&;-v z-EjgG@OT0V(dG`4>fU0tPtLVCS|MO-len21R>u+}F1O@2&S`L2PgD^i|pWgi( zK$!65JZ0GiD&&B1o+6ac!{*%Gd<4|w`Sehzc-xoTd#q|Vv)gG?eEiK=d^-Y)`3dG= zyn*DMYy*G_79Urg0}p=UWE_w1b{{>9nf^?iyW?KQzy@^-Xr}zu%}#z3X^!J!a^4ib zc>)#a=6e3y!?bY;6ZykC`CqekoxDT$Xgir57*>rVffEHT315@EQZb znatGY1^?g@#kksMAor4&_1W&c+1;z)U-*~5M+ddcc~qiYxlU!QY@{}d$7wvC4{k-d zm(X{ZdMhuX{>z__fArW*tUrfVm5m!X*eLs1DazAYZ|E)rcilhoX8~7o!QZBH_K~bF zhl&1icPOwh{uIHj@hJmY=-NvbmVIlAdoZxc*9ZbIAEUd4>0e<$!Q%I=0d9Bo_IZ7r zULW`CgQ8Q%!(k_%5f~vHp!DNdFOwDIBp4xL`Bf18bU%s-z@2q{m>?4;C{ags`^3>J zM@rm|fDoL8hx_$be*6VnN#DL1!ZoTl?P8j}aHuswT+j|-CjXjujA_u%@~z&3o7S1R z&(q`G;5|96L5HT52Ln>uz5mh!cjeydGTzzWM-^6s#_|I&yrwoC>zwGG`pT>N0UW`? z*-_3)DjE0z2!GyPi;JXy;j#^PjSIDDK9^eVWEMdn(`%DSp& zdGD($+5Y@^D^X&m5d7#{+@Sf_h}X;)khS2)fkmQBYXDrfecjUo+=qk7+cXyBIG3?T zpy`4V3xfy}t$W94)^dSxgTv*OGMd$l0P&}`H?VcvF~P_TyDVjCavnpEwP1mcvV3F# z(Lo6tn2S1)s)^epD}__@+j|{0=>2++ALhmlgRjK@*L5WRe?Jg4yE~rzc*%tY^$m0! zZZH#ugh4>|C}`fg!nrbPP?Z$c z#}dIH5xZrV*!xOq(PR|ZCLEHpx?V3kQbr3V9Aw_8t*VZqP4yeSMI$=~;RoRAgkjq; zq+{kdLejh^n70E8a79 z7A=aH84&Nd*J0UzGJp^r#9Z?+HtMLAD3O!r&NuKDT~%@z?%j27fc}JLt*m_7zY<3mq$_SIYquW*=1y|=raXE3~Gta z224}3Cw>s<3K(J65gaYx6`_tcF#C>dLVg%D>F+7DsX-2kdptV|;PGa?37N#E>E%nk zba5Uc;Z*~&Rjw;WoC%ctI_@?egC1Q|qRpR58FQ%@Js-^c5v$@2C^DIZGbO`M|l*7h!^f7H@%Q*#vq6KpzpX!RV zk~7;d%}ET(gHa6utIM~)TI~+A+F+HT_WCI(3HEZ6#Q!x6 zx=N%LQHEks?xp^q8JJOE7n#0tRJE(!E&_?G0b0u$u8w;}E`hF>#XJ#G`*Ar!A;ilM zk)WSCn<<5Cr!nBy!1*%o47e(Q&l8x&WX)HsUN~d%AihJN|9J}?H*bJb(9*eob~#zK zKx{$VvnE=-NeLF>jBwIEj5l?iVY-|#9+0eK9tof9^9HhAW<*|>xgxa$aazXB0n<_3 z;DM~?CcHV#rABx%2CFLmX+V*MPKg9_)T1s!L2F3UsSS$z)a(Ey00sTnZYXU}+e3;( zuwehJh?XHp=vvu3#Rnm|SDCp8Tmvl)?XIqjwxGweEr^H6(@q$YEvWjKliSU#8W_NF zNE0qd!dUSZ8kRZ#X&0WRhEx=vn@{Qr%W@rBmcN7E)RVrrN?c`{t+_AA%3`ZIv$cR( zSzwI0%jsqpHPiq}lTCt&OJc>#S}@I1=yXXCW}#d|i!f~e-lnu|Mv_&cmkF^N2L{QJE=px89`*HbbxFDXO0n1)a z@r-(?EYrIve%3o2GnMC4rLIoC2&NJC+nQQH;+V?M)Z_tH=f0w;U5!v~_ZE=BV_k;v z{;*!1H_;J5hiSzwq02a{0osW;4d*gR!NdTbol~k070^E#;1t$GKf4V9g#MTL0t}liR6hLWWlPd;^++cXXi7e)z^m1nlBGnGI7Q765z$*bK8)c` zBsOdY3N7wth*vw58O-}1WuXNAt{yYO=^AzY4D>R4OXZP6;Qx|crb7cNO?+4(;2bq1 zFnHXAOpK{|ak3f8azE{Kyb*7mc8S?sml8x(o^j9z(YdIt8V?i&^} zUSMS`Sa~^m0X0h$vg^@;!0r3Hz?~%lrDcfh!)_DLdLu!s;U_-sgO;G|L-oKko>x}^ z7$WE5kcqNZcRJo!r`6ac!+)cA-G42mQ-C7Un}ji6Ishc&Hv`b73J}xat3M1zBj(=d_;9) z8a*=M8|SfK7uMqdijPawyQ3LQK1V<0@w%uY#nX^p=~*!jpJ_CEU36XT4aK$ahAuS^ zqi@HGqwjhc#*$Lmek80_(2EbaFUq_mq(`eUg?twWiC^7{mESu|9%}%xM%?BMWzu>3!TnHiJHGfo zIirDk^WyKu@waU}w_o5U1McO1THZiZM{K;BV&i~O#m9};O6)rP^H-nv=NyYW*`bQj z1s#rs>byO|rGHa=Ij$ZY=@i}%zDxXnnc8Q6f6ot-e{;!?7xJ#Bqe68mt3Ql&ieG$# zWe%8=n6G!Y4n*U~sPKLuO($y!qvmN&s54|c%SN+q94+c|WWJ zZMy^co(b{-9a1@RsrzsNc&F5K@D|#witPY)h#c{hj|`w?@}y!V(V;`YVV%pD1qv+i zp%ChM1{n|ND`W>Y*4Ku<&!`Nf$&f)pu#C$0qBek+HBikf5mu8>y=NKhMb@mB_%*|u z=FT%`Ag?@=HHf)Tq{}#wsTJ(5x2EEUn2`ugpHW1-o$iB{c_%JV&*4%m#B^27I2`Xb z_;K*bFnNBcDR&@bXL0ga>6MUN;vB-ignmnj_!>i3PmyBg4P>&J-r6N4$Fonrwh}3G zb~UNd_w~?ny?G}3=}o5Ir>B>IQ+w}*3Jrf%_JbAldUC=A+g`^fy7)zzcx;ff~TS{YIR4q}uVG!j-NxC(lW21!u zzr~F=xD`r~9QTLF--FPUcDXeBD;Ba_b6!dPL~jGKKZ2;JpQjd9wmJ$M6C^-cm7LFx z{=c`82$wM=>a05pM@~0T3Ug6Cuf<`IuZai_LQvw30BJLHYO15}O}di57w?++@w)2_ zfAqm)R~wn!j=mW9?tUC)Q6}hm7!RT9Y6m%En5!<9OrTuYFc*AUdaZ5A%Y6ZesexIj zD3P{|pqynTlgkXA%lxnegti$7lca8>H|bnfm9u4w8k=}WI5(&)od^Sod)M}qq?${< zAV1mlU)Y~2IT#;#qmhHTwb z*kLbi)OCY_ZR(2hSn=bvC~FVwPAt%v7L167Ce3a0Y5>|gD)+}+h&QrOye8z()m1!> zEYw=LLG>|+pdpLOA;2*rQ!Q%f3@?l*8;e%*o{9r8`y5^z8uD6BF3X3x-lrf56c#hP z?1Jk=cIuYN6*X)MHqd;edee;Ew5IC~a|=4xG7@9RYwXE|Mu6+b3x7!-&mdTddO+?5V@~v`2AZ=WL39Zp3~fc zw?QjnOP`m#b5-9d#c?rQ9+)u_(Bx=X9wx2fK?55z-7xRUz3elgBV1hD zF2@6t;$IDTNzI3qdJQN~xPq79sxfN=t6fKO1S4N%26lXNSV!D}fzl1yd@?qtajSES z@=BMfSmrBVMpxhFMAMzF%NT6OOyw$EfZ@m zSd9#5{bV~JcAN#+&=?3AGYKVC>BMz_6ZH4#o&0~2^!wFT)8E%Iv+Jc5&=mcq_#$MQ zab2J5^!}Ot%gwDzO#)(baG7YI+65QuPjX|LhDQ7CG6(LJA}UR-R$WX04HJ~@4dN2L1ki&+UB8uY{ z0u6`aZgW1kwnlgBNC?Y+qmDGnY(8rErdi!Z@ggMK#mzmtX4B+scETViKRu$7(!8;M z`SW_KDR$5Z$q#N<@m7%axRdFW#{q?kJntlCPtz8SiVv<4a;I~Es5R-leY-{O8EI_y zQXl1GoKOuIZw}}E36eD9(~HxhaP^AcCH=2*yT}$Ssuiw+M#PWbVm2HY& zpEpicWAIngE?xYcyZ~FFrPBSW{8L%JG07j7rubr7c^wOJkq4bq|9?Nd^(>W#*qQC* zQD#CD{Hw3fp8!8Vz`we2Y9yE~t++Y;8>@$xVqbPRI^5-GvfEsKnuZ{k4(5 zckzxq7V2{oi1p!!p(#1GFTZt*+fCF?J{`Gu7`; z?jB!4^X_*+@#RyMM)x!8C5yDLHpmk>#_Cx#20!p?PnJ`B{;)obtLtd$WsKlQUPB29 zozVBOfcnufT;E;L={w`4l)eDw7oOY$gux%eupF%Z)?}9f=dAuQH~=KlD}K4Y9XIzo zCw3J0X0P>U5_zS_B2a5BWg)(#VnQ|uk+`bu@}cGb&3}--J3suiJH@r`|2O^=)t&G% zbpouR24r{L;?b?qV<Ze-QWAllqyWQ=~8po_5HYa zjob34Wm|=x`umD8XvX5ySz2v@`c_F<^UGoaqVkS~$L9n;U`EW&GHbI~P+N0;KdH-D z(1y|Z=yrTRUV*8H*}#_Jj_ZIm9FNA#ENTh0WiJ$ay&ndV7bP2G>*+9btdfzY_$8j^ zyt5{)(R(UxNuHuccji|xR4=>Z91CNm95(H_W4%DbkS*qa^EZ@0EgZ7pDUt&Ol)bd)#Yly0I;v#CAIC>f88BFl-om`}5 z@#sa4ypxZ^a++q&1$7XR=dQzt7<_dGjO^4JzsR(<0E<%ld&sUq{gGK~((bVq@~o`V zvFvgXBdEM&zR$RDS~Pfa;o|D*Ft-juD1rGK)z9%JF7d@A_1@|6CXmFT_zQ=R4uj>6 ziHQSz9U|C_BaBs?Mm`{>&+<3m4Um?hV!&KIab5Di*-L*__9RdS2sU*ImZA~2TSD`FX)3bo<)@L&021=$d z))2N4S5Zl)SaMSU|_oz)ww->hF3J+aYsAv|=Gy2X1|4qEL6mTa?Wp zQ8uMo9-z(eIL^&#oO_~_d~Lc+ArV1>4%dJfWcu~zGKK8KRmac|byZ*aY6l#dY+~<~ zRyqb-)10JilofQ~nR)MmcM;TjmErPO62PlnRSpbVPnwS_aC_66US#G#UkK&y%O5*M zf^|7a8zv213W$M@wPNPV)q_9jDgYOFvr5$#puJUdk_iWH$RW#i)^OM1BIjxG5oZWY z=CVg;;5ksXE7sV6XJLfR0+ubGxVRU&0{m0xOtOF{ut8FNu|vx)T8gX(7s9}tHxWo| zk*sD7u(bQo8rixHSI1*hxs^%pShea_<7}Ta6>n}}qRee%+I0y}t~D(A^^sg3$k&r@JR|av4C5V%P}E$7KvLsP<>Vusr(rDL>=)q zJ@Qf@#`ROv4xAAC$ZZUkH&4xK1ruw<_sGnCM0d$@sWV7Bq=0qYxUOm{<8*_zNHpLF zDp`$k=6VN5ZPnI<+IXJ%J(~(_ljJ5Pr5+RlWWCWoFCKj8?lkhO8J^6fXwh2Pk|@@{ zsH2;zZ!2i;u+Nr~)EdjhWyIUzk(}5KbGa^B*igOz^AK^3^j)V09k@SfZ*s|6wUFbF z3nYo8BNl-;h?)Im^$=zbjbZZCm;1{?zFvL_LK_*jYh)pUQQ4!oUss9EUth5JYja-? zSiVffIX22o;N%>O?K#LFOKfvO%gBD(y6o$DzuUSr@Vx6?`&Nz8#7rKt!D$8%XsFX` z{R{lSG7sqF$E$kr=PswWbA*)oSCrffS;;e8yh0foo!q?Rw?1f^H%&3)-a#sQdfvVE zH4-lqGuIm&mm%sR&+F~hBf*RD<_OWC$x}8WPp(Z9KH2Io^soQJ|B5{Ic=AsR-SW-( zFm4j_qXeVKz()HhvE^zz@{7g8xKYVzH}6mn-(F0Y@{LZ9>`o#BMP*11i2JT>igM03x>?pE7BrtQ&Ho=y*=FPwzw+q)@s zl{3tRlZ)Rxy9S2!R&IBm(_Ql5*RC8>tlhtQDQLI zAZj1W0wJ2Yjw{uZb7m9O59lJo`*@Z&iT`-OEcynHm=X``F)Y!KS0R)A(eiqi`D=Q` zxombws`H1Ne(2#7(iyqTrE7DI8a+SkWV#RN=Gb~7+0*g|@sGL1H*XKJF`VTEcx6wl z^V~f?Id%x$M6f6_o5-8uQ+L-Tavs|8;@Z*ZZG<0lEm1PMxCNavob*~2^zZ$1P{d8B zmB<1>H+wNA6|g6KGf!&}jA&y|;ljpH3vuMJ;Y08rJxVpXeQftunUej5yn6il)~~^^ z=KNjmToJi!X|;KD(s{!AMH>sQvOc)Qr%$P^a_@fj%>{D%=i+O~aQMgl+a%?Q>cax# zz<`=i2`L@!opZT!npG@p+i`0C>Iyge%FC+v8C+KQw7p5ySDd?y6dxs^nM_fdZg|eB zs>IVHV4tQPF)Xom=1zUm84V)cz!HB^)4T+EJ>@Wh$`{6S5aLvoy&d4~ygvqmiET@o ztGOP85UQhe&73@f1$yimm$(qIF9GNy$J?p9=y=&=gwvC94S<-;ov1y}keKhPj^4!4 zXK+!@k(ib#1(JNWQ-Cisi0nBt1+yUUsycgKB*wIkY6t=R<)W%Jy9`2{*+$2Kk}SSKY%!dGBcqdB(bW^aO?!tpk8L7 z%+LYO?5nUrY|kj|gT>9t^h3X{3^nfKazI!IxUI9pui5$}I%|$islA`~7*g3Q|lAWKTp&+a4NI z@8@`}S}V(%vK`p0??ci`Y|6f-z2ZOk!3JC|87TY_2v|AAU!qybewP@_{gq9#yP;3j44aRBZEvB9Jt5hz-rxwE4jE27T80XFpDXT4lu5?<&qm}seu zBCW7k@MRgghs|^hOm}9GbAZ~fXL>oyNjYLjLTb8WiC1-$Q9K#ImPZGlFDB`owBi)i z$bhhmFQ)p2^s|)2kW9Gyg$>hrP(^sVtq}o6Rkk@-SE&`n#;P#x1 zPYV&?4rj=Q^N>ShNGoS8t?LUxG7ErM&7n`CHzr#}S@Dv9gBlDPIbOITt=SBjH}|Oce&Ebn)(X6o9xtJD#r)ON=^-?yiiO0L z+_8{q)PZ!f<%VWQryUGHu|^jWFXX2w-ir9P82N*J&XeXRb`U?$M#HkPLEc??r7&L7?P+|M(>c^;si1D62 z^|=NsI}Q#y+O&74NU$ZI7Sa&R8Zq~xps9_8(d3nEV&$inP115JK50!>;9V{06{i)v z3cQ;wOK9&-ve||L5r`kT`RE1?bYT60!a7M;+}(w0Jt(_+({+De56K_4zuFeR_8^5s z$?d&~I|p??t{?ZCm*v04cZ8mUT`~O$$BBo(`?}5cGMDLDG|^vh=aQGNcMkr8>}DV* za>-#$o%;+aehevDbpImHe>}?~yE*z_iHd5Me|C*3$5Hdf5W=1@{7|%pnp!+7!hrM9 z!u79i*V| zyZ!WVkLc@+;?Wk@D{epf&HCtx(d1#qm)E~vI|h{3-%F6^xT3y19(P->FXu`nj)&>? z*0ZDOtSJ24iFvILNB|r!wBf#;!rN9Avc9Z?5>{C!oN2OHz@B#vl`oVW#w0=RT3p`9 z_6Qy$XrUJVxKDul+vz!Z_n5fb;C*=QvN7I@V`NKk^>(h_UAtP#WmUts1+5^nw+)>X zAxRlttiOO<*1wTA#b+-*?~QeJGakV_CL#TV5QX0>bnfc!vUV%i^e zC%>hJpGsQ2+_WvfvF}F-6jyJH-^fJC4WVr#6ldlP{-UGgbjeny&xRL>ejxwZ#`ox2 zk}h7()@j=7s$9M!UnyES2f~*(ZByux7JmKfFTS5!TXY5BM;y(W+ePIK+}=;8|5*HZ zmfLUVg#aQp>=81zBi^*DJUHxBag&E72%mpwpOSAGb85Cjrug~sr8 zEjN)s*Y66mZaFx*{A(=A*axJmDE_c6$LmEw%bFq9t75h<5E0(4@0?$Jio3wy+1Vjp zRzs&v$b4G}taq$#5UUwh`~*n0Qa2gpqv`C2s}`-e?5BjwL-(5;QQc>~h2ezuBlhc8ouqTY^B{;}ObhApT%F2)}`2@=}fD`pK^^t!kP z0ETo02f!Pl8-Y?-NiI(eO0H38h}5}sFEuVLyiIRd$h# z1k+fY!f<0V7=B&&-b7}C`ze-y>()hbL=t`unVGf&Y5fPYY;99htI*N|=Bo_Qz`1W~V!?6Ad0>5S@{P)bX0U{QR6ZkC-$+s8 zGF8Sf7sJ)n`4nWrddn12ej3!EY+4p)T(n`K0Mpl($6FArY5=dL66r;gLj!y!Jxa`% zH?=H1S{I~p-qcmabeHN#dW#B7NZbkN4tm|iB{VnOx094>K^-(ue$rpV8Pr49TBh{7 z%;(jE+hZ@yVr?00WwoW%zHGf2sydTOh?RC3FXiN;(?sDF=t6?N(}$b+oXR|LIKqSU z!yQ^wD66G0j+)4eVcEzY#SHP5KyQr+$1^Jbm2^sB9DCtIrFliR}5z zz<2BOO*NXfnA1siQtXv#(ji4h?rsXvuB)L$o8{#KNNYsm5zAVnqU4f%t*Tf|!+-pB z*~*^JjEYH*8l+5EA9BHs;0!BqMrwklNWASwdmv88*%+5a#2zAgd&g5BX?fC4LH>p9^PqTT+A|qy1 zL)$}0RH#V}NJ*ZdRV`#k+u|g{Uhox@QVu}L{P)wm&AGNUU@E-6-5cBF9)U&c~Y$$2EsjA58OLH zi;UT_&velF6Fc56i06p#?J?3Qnx>UwpXe)+8j+YZT|xy(U&v_P?+7sM=&h_%2m z1Z#1Y;?{P|Cp<2Lb#wq`XxcApI%QLFb;zZ|r=Xq%$v)-^yuFR3P#|TggqYOAoEkhc zNMO^yc?lNQavH^#T+~9&AZ{LWtRiViv})y-sHTczAsMH*LLnJW`dm2yIR}$G%H(Y0Hel)?wQ2k}hlfRx{9sy;Q*x zZ%v|1I`WpBAO>I&L)jxhg~P(M74mTGttyznf@Qp>$UOlz8;b*m-7(oxw#S$pGEQug zCwkfP*ju;Dktc;3qMFG0D(5h5qYD(0!2*$~9uk|stt;}ssRGuowli;^uLh64s}XWV z5povFhX1%ts-nL|%@6i-vmO2jNMKyR(Z9ju^>b*=bI-K#&+_|m^X})vdMdtu_w!2b z^C`Kt2&1EN4<^Et%hQ$9Ak42X;2T#O;FpuXZ)M#&QFuIvxRcf`>C7h1HXa{ znHEpuzg!kiKUHMJ2vW68THcM+dZFcU9{lK7Z259;im$YIJN;=|oe>OsEQ$D;#p(f^ ziypNi&+dBgE*i(&;M%pry0d?U7w=IGi|u1MfvRI{d@ZiWUa`;%^#FL+xIbHzW7P~ z3VsWi&DZ03^`K98-b?i1ujF6L50kt^h)zw@DPmK;cPc-?{Cso%lPuKT`EajFc}|Dx z5gV*{%#P-BaXBKb%wPF4x#CUeFQnHN?rH8tejU1c^+m>2B9{$IHTE&m_2T=#nL=~q z5$K5H+0lO!EV zJiSthtd@P4v@|9*zyW8N)9xd(ph9RyF52AfS%Cn7u5jMQ3m`uDP$|hNY}JOUp$8=Z zDnV3)38&0t;Ik|(`{P$vM=hQsp-@qJ=xN-58Q57s(~0{yudY^ypjXvr8nx1@S_vWJ zuHi(M-~C@{k>7Bd>c)SyYEglHkfzFiX|F;LWKBqmxsB3-9G)KJuvYF*113po8$7zb z!TDS0jOgf9q}BDIZjE@f)H5d0TEGGHP0u1Gi-@pCd`zjuMOCG5x~{5OEG@05qCm}2 z)J2>u&kLWHE)>#5U5~JVGBmo^dXuh2b~>}hg2+|ZIAL=N_xxt4Hf7h+I*s}J&e5f2 zmKnB2Zvl#sh8`sC*=&a%VnMuAPeID6V!ee$9=?^x=RM`f@Eg5cg|pe-X;2FzpX3Lq z*3hy@+h=r6)CA>-46bL2xa2lN&EgM&FkBjXXNv+-TSF#+xh;cp>Wz?bW30v16+8g7 zkZ_Rcb1sUwZD~8+2;*<*O_wDnNR#4fi|ju=RDuM82~alNJ#~tOd*lk}a9YID%>cOs z${j6Q4;%D*^Jlf_qeT6jo53!cHoK-}@Y#~}zv-$>QrPXT%5e)_2Cn$%Glf^#bY)|C zPkEj>K6}9JXlpwkXYS4nncYP*P}86%mnuSgI6&T@Li-w31J#B1CJ?pmDV2VaifKv< z@cv~0y{HADeYh%VO)XLo^)Se>kHcA@m#fE1MX9mRuSlg|K}LIJaG{m*=jYhz%x-BB#d1e3z^5BRdD1to4Bte7huI%mvs;ipy*c+Q8nPzvQiGDyc8O$`) zP$S__?w80{ghym#uGpJAxVm~><2@vh-j$h=@o@L$z7~QpHINOKyaAnKx)YRrQ#QP^ z+;OjfWbbpdLW3+f-LF!__{f$K7CqsjP1CV!s%~YlscPOo z*U(V`{lAQ(?|sYAkJ>}?hwD4$mKfQDjb%_+_bzYJ$>~m6YmElg0<;BGWRjNkp{By{ z+)X*K3%QNuYL=KH@kWh2EW3#Nc)fNw*ZMZ&u>0#(yr!@n%eI2Pbdsr?dy)#-yG%{5 zE%EI%gl!flRLM$XDUN|8t+)XEQZvSC!Gn+;U2gyiJjxR%IZd^~2}5Q}0HUz+3E1I8 znmi7SsU?7909rf+y$+EtayAzFwb10uh~PdS_fcw7mSq7wM+y=3YK7^-UF%MVGhnq9 zDm9p-oJd4uFG8_6odnOr90bW0UUXa-vP80Fa`wkR~GOZ1>5c6W^C_yr;Y{6AaWX>~ltpqR?qc^V%j-ow;zVpNJ;8OR2 z)z?aNh)R|ZMKn5TWQb*HQ`chfPSM9NXB-7zr??7?%<_hNk7Es&mG~aMJ)=@j(YcV| zy^^DV;-WFqy*Ai`GIeIXUUzj`Ori8RGm40=m~drK^b!9`|%0)-Mz&xX+AtUHri2$VK{&I-Bx|U z`i7X*$k)TgfAZ~%!_zPwW*Ff9xSKxqj<v%Z3PEfqJq8IwP z+^a0~<^l$+800fpXx$Je9-${(1p&~0vnWH5L4F!!%zY7i3NZiW4_|)!#W#K?ev=qE zkn_t2Xq3s&f01@;<4JxaciA6bU!2Yy{`RZI?Y{6XiJR?5NGm)jz@xmE`|G{`=P%XW z+I;&8oaM78EaBWQet8&APwy87i^pBOZkX3@kse>4kB4bo-OmH8U+FjWJ?Y=dGXPT_K)Ch@g3RtJO8_L`}}}b z^+-OdfAh8O_Tiu1(@$W8{p#Y;HL=%rd<1wAyo@lR-Uke?=%zXDBQP3=#P{cGaj_%0 zPn-24y8yu9_-1?m`TQP*zmVmAaudY?$df~r=P5tW!{1Kw{-2*@zu7tBg7b1UiG{rV z9~}bt`{Pyq@%KOf^a`=e4{f3TGyNi1&p1{O3a8J$iL&cJcJdKiPu$y@bMy&nSGC~m z;0`QJLRje9i}RUHTl?!EuArm&=H@kYqDtgkGo4QGvE_OLVpK{|PD-o_ex`yPTKWCw zAp6!M&>|U(kXHv9S7g{2Z*I<$Hslsi#G#OIyeLK`kyyd}iUC@cIHTn9-tYaP>Mfw3 zLUc1XHwl?((L(U!gSi8zO=sBUb7$lm-wljBC28M{L6i+Gy(r@*O!9aLe5MlGksm%o zHDZBmB-1##wW|g)j+ALZn*YJm&l<6)DVrdlQY)ew8->FB-#G z-sSQZPgbM0wP<=)Im)!iLL{f5Xft@9-lXo4uKi#T0&xH)RbBux9b%nKmv|z8 z*EBMCR8X2=YHcmcg1e#?xq`Nq5R*0lF34<=p=0H^RXb{H(5J;~F~JjCa42xK{=Gj^ zoJrI783M(mJY0rff~IUscK{^;NmGJ<(&N{gK+KV8K@+9x29YmiqV_n}{B^WkQ${Vg z#A+gk2%RzSW<8`v7Cmz`x~UR-ks*|_zR8;E8-{47ph+d}wZAN(w>6Y!Nz%bh3D}#e zSf(%<&{?6wY^j7;fI2fvUE#w~w7HjD#{Li_+3T)B2Lerd#B1K5mEr6tVBQBvuj%A8 z$gG;vB?bOu?r<8yXBvWq%-!V`)aOb<^~D88MW{D{x0k8N7*wNB+>lIL zukIJ2trvJ8Yh0x$5Il`c;!q}6Xk)HjKP@M>)1bs#nhJF$>>Vn+)yq*;(hrRqp?aIP zmB%3%EfRD9?W>rcb;-;t-jxh4&-6uJ6`dN$I)CJV4c(LCG87pJ+elO`Pt=qxXHK10 z5g^7Y$6u0Wv&zc$9dnb!`YeEPU28G`^+S*w7N^j5yg;7=a|rrm!GB9rN}QVjeD!FH zXQ8?wroYjsTW|%=5~XvJ*1SgN+-ZT0kqHzeTs({8GB+x=4L2REYG1_07-k_%ikpg7 zHev2Toby!PN*=l46_qeD6m?Z*PR9gRas0CdD>kL&7X{j#4U@!3IC_W{nX<=oP^5L# zku4JrMPAMVl(9Uzr&;z{_66C!Xcv7#Z>Jj6&`iie^J-wbl5Vkrq2(EhY$FR+Muf8Z zCZi^~@b)YUlPW6v^BD-Wklm75XP1(LorKri-W>oC*FKslf6}W3`e7M)ep>Rxc96~^ zG3&F>MtPGKns;OofV-w`Gm~(UrWYGiGUd~GiqeUW(>Ckx>$^6fU8S!(glxu1(J_mIPDNITb&w(Ra>Nqj3^BopWlJeHmrk z-$sN`qK|@R8eAZ#x(78wiB?+3HIwgnI-bLQ%DQDFvRV`%rCDA7FZ7xbub0AR zHzmD+d>4sjSV?df!A!WbHPdCKtPQ5I*h6WH+>fER5g4?k#gn9Y)y%Xx*1kmqVii5!b+U-2Z(#hAvS#eo8J+@TX5~{p`7q zbZ$qreOW`|b^U?HgX~V;L~bK@F>RZe{Cfz2|LmmA0A7kSFOz9*+j3kBF55k?*R6`o zJ?7BJJ)Ih?|E2WUepA%klK^-rFE5-+*G`2l)qC z&I+nLkMak{*UK2%>mpL3KZWkPpnrFM9UWc@;K88*r|Eccnq`0)98P*VzS}MSO$wa^ zj8V5OM{c~|Yv11&;4pnUpwsJ5KmVkKY16@}$8Em-`6u@?KmWA(^;b9;uAyRfpMUa(#!czR+hkox+bt1n?a0;}M^DqYnZ^ z9v1n+_1Xnc9O%kPZ92F^ zHLm;1a2Su*^ZvaXwe$M-AX<5KVwbWd%cM}|W4V5u*M_h70#^DUOG#E57=I7095sAH z*Rpz+_Zlury&c{!jIxY^Tz_={;a@NaxIbV< zfOwNPksp$64iI~M-9tfi z(D{U;9yfTJ-p-xe9+7>$2H96XvX|p-ka_dj#cyX4YE3dl@5e{Ktzu>Ec6$eppF@|f znE^kXy}<1Kc-YNtm$HHVvKB;Zik9`RPVO4|yxI(zQR3=Z zbUS1ceF?$@X3mBy_DA1pR-*BR0$7|ydf8@b@DeDrEKt+UEUX7*47L^2+gF#pHiC6U z#4TmKKx5;T8H)zBQdDU_$M9kzE*%%_C&%3z-}@)g9JwbqH?N+389c6B`E_&iGB*#l zg<*bYoQwW)QyI=723bY%vkepfCV(?K2DzCh3A~-MVrPi7!=1zaI)t+LJLrDqP0%Vz z38l1m5fyTokF^^VFO}G@xRM~jj?KrB~(85}h%>!?naH?g%uh3w4+Xj1}!jx(Rw8uWT4F&s+=&f&VO z2+KUw%R|l^N+4ZxZd9Mvn^Z-`=ngzXy=*1U%iypNezd^d#tj#N5*vM$b8j?ievW836d2;TnyY2`rxkfr-m0bgI!Tju308&D#is_AXgZCz2 z5ayUk-!<4^ZqW2gx_5z`qEUuwzVSBxIyz8T%_ni=y%`W3PEI~WWCZMgoSa-hnb2r! zDNQ;bnrZ~CH)hQ8QGlA5%*fo_+y|At)>H*6)F6Oxmz%C?dtNh%!eu9+D5GZX^!5fQ zb1<1s&=>1fr#>C?;d zXbKaxo3V}a%v^h zkU+Y`J@-}>C>=5zu3l|3A^-_@UtK~ykCy_rOK$*HdTuP0U{hN?#p7byxJ9#=JYfX} zv~2D=0H5MD8ZK)=$m*{HTcSO?D)pi)69y39#=WU(AVWY$Xsj8iz;ev;U9Qy88&r;F zlD-%~&cN(lS@BLA%JI9B&d?kw>#U(fa}4X9QQ4zko=G~A`URO>w&s$;Rj(#0*3d%m zu^bl|fXhsWR^rt$`L1;+c2u!WTs zBv`3*LsLJ&#I)oV-p7Y1O@`um&WM(%h@zoQ3noe+T_X-;c_!!9u0%u z8_pvySgT7Y9nxHoEno^;jclu3W2ltLN7|tcvnonfq+b%Y_Fz5Bh)c-M7;Qi(U675Q z%+uc@cW%Xs>FTybtg|)Hh@+_0&<1eDw7EhQ;?FBU8iOp(-`fm zk;yb_h7}T&O3Mz*me?7Or`Tc%iaUaJrh`jf1SQj=20IT+Xr9il#Ue_7g8PPmV*feS z0=2Rf-5sXT@`GC5uB2KC@ou!|xUiLh9yRaT35&8kEV?My<13Cw#Be1VW*K&$(@_Me zM}IVH$|q=wOsJ-7$&u16QB92O{!G3bB}F7&M*AsYfTMiJ&|WE1HzTNee7RR2#&%Uz zGNtHge0$sn3wcr&9bGbNqmQ#=aq>kiDqOmP8BqdLtAze_NxKWEDRd@T@f;0jNrr=K z4GSjVM9$IAgl0wX9vA}pQ|QUIkfCMr_TX+wP>Mo6rbs#jI9i5_B8KzuI0k?bz3AK| z>JHV($AcDSyY$^0aG}@hmPrdv$NT%B$!O*1HZ+gN;dFWa>vi}+XAtDWYsyC4+=RJa zR%OSlAt~gvkgF;PQ1WvyaOaBm$t(8-hwVc^O5tWS?C41R9A;R7X-XS}D?JPp9q>;< zsFR8ynD#Bw{lCb`w31&1>6q@L#{YAd^EOOLWhE~ueIY)m43mY8gM zd{h%TG`OBh_URuEQTElKvV^rHz@vORD&MXJ{V5Be_;-(vzBLAJJbUp&$VM1wK5@Ck z0!23{b&*NTXW0S?h7Vm5i^pSlEhyQF6V2h%@tvj%WG$nFi&tYHAqIxnQ5o4ecHoNt z%^^urydV}plex57kqvJIAr5hnK3qnrW{|?o2wVe~Q;VkQn8REgy|)2=m01OXW)eYw zDxzAH656tE?+{Sw7R!VZr7tcc?=6}JVLmyN(+}agoW+*5@~7c+n!uyf(Kk|$Zs02yD;lZXz9=q;m^82-y_>LWTb)5VCLZPe(_2`VLKp=&kgK-9PyTKragH8w5ieBR5xnCjWjq?Ht&uK%h8_ z?FMwmp5Ao-e)Dj0R0aOehwG1zfa_FuY|Qugn!d`%^U=o|nKQN! z-{J2>gl&6yRyh9s))RHh>UWp?2lD~wtxG)1hnSuYzCk=B5jM|W|2M_ZH=YLn{^|j7 ztXR1#I34|P$$s&F{Le3&Pnur-`|m0CoOm{4G| zlxD~6>=r?&X{%pWNC0q`Psq@4TUSNKlXUCRUkCR4!Mw~H-}KxR754*Oj@q(Z)hH(X zFu;bcWLtFO#dKclZZ zXgvZHHOXT?jd}~@3Fy?H)rAq2FxC-!w#CP(2v&#_`i-wskiXA?=Xx1c)!dDI)uwbr zhOAeU)N+6bjIno|tZ%eRp)V+5lcc3#B7r@{Gy#S~6+_o$0`k~T-4vE1?T7-phU=Q! zspH|~94BP+nSL!Mi@`!|S!j9=pe1}PDCv}>10{;eGEoP~G$+!+p_V(!)WBk-bJ=0N zL)puFvdWEfS=6Gdx#xCvf7A%{qI8?NPC*8{vy?@Bo_kQi;>D#DZqL-~bz5a<46f%; zNFlR# zH7I)~f?RU;wgw8h>-Ts!#in`Jl}F>e8vyCDAAP8W zLDM^-+b#t176qUhnw~*Bu7MD6OLOb_7z&~qv{^AF;jB#$k;aS=o_D#OMUC#;j*?rT z;Lz*HQ_$v&z{Fp7QU{zEU$tk;cymYm%=!zdO+ydiJPNOiU9udm?5Lp)zLf)p>r>u_ z4)Utbkeg~`JJ)Me6?6G6;QhLkfnsn}qd#v_bZtf;I01OXqN!kvkd!fp=fm z6}@ee=&hQ%m7R$!qE4NJ6IET&N}t&pDrgB)MeLl6u4-i2=s4HJiZ_v$n{Ftqh!AcXTe~V8y>L>dx@uNO48@lBS@UfrOD>2LTan-s?~hS`KU~`Fw7daVqh~LCT+hjon$c?T&7Te(})kojitxEV#e~}A%{)g zMK~b9N6lr-DV=&#G)z+tn%K}?ERJN?@WG#g{W98R&-BB{n&@?#kt2VIGzDmnFX$x_ zK%sPXE(D`Jk8_8+n8KvzXp#(eRk};kv9wiZJhE$G;o|axn9m>JzMLQucL0 z2iE=cH1kJvZPRhJ)BYGP%Oc=**|Z!1tn^LyF)}5wuyRPOe+213y>1|6PJP~?3&|Vd zykeYGtNDux47bc8@6h+ZC7cmZc4%pi45uK+jT$nB1TRP3>%j=fUvKECi#+FC)FdGy*8!cM zkFT#Y?>lsmsmw={v1($Qj4Jz**WKFMaPvxZ!_>$$8D91T5fcl@$WYa*&(j8zg6lLF zz>K$p04h7KkH=Ws3PD8MrdvN;+#4yffJtiDj= zB>v*sM*Ly@{s#U;rsQu$-QqQPsOvzMrk0V&P>APG82mZva?6QQ_DnZ8H@m^c_LY1X z%Jh2JG}uT8%Dqix zbT!x`>WWV1^SnTetk|AwsV>vYxQ~U8j?ucUXo<|7v9f-k7i;L0PbZr`JF2cM!wiqE z;i5-srEl9WZ>!|;KVDe$e+>ry*^Abmpjyao!6hjRck@YC-}M0A5k!2tc{jSU4*9`U z^}4s)uBs{_c8Bj2n07^!{cIoeH>B7vGfT!|*KRlv( z!vE2q-yLsVXuZ(N{`DE?`bI;9^iw$Y5s;!C{V2$vgCuG*r+M!JV~2Oe@k{ssXa;Zv z5b?*rgQA~%=DBbqI_|v01B9>VLKs=?CQ6h;h+;gSJ@E}nwr+RhZLB*f?n?JP&d0;8 z{MT)~XLzc+%{L(FFUxeuJbYY#~WX?g0#;1fi<6p%i`ivWkG^8qpn3xZt_W!${RGl7t8{w#ajmQ?G#oA7z4Z= z6*rsDB?B<9wIR3p6dw;_3zREzhl0P3hok0=xw#%*Tg{(i3IFyTnsGnURMvL|SQgih z;peNj0Nlcjc;QI*HWDEiCTP(0?oA|E21Gy!Hg^hm6N9Fp^^oFx*2`U{6m139fn2m< z%7I1PBf(l-Qc2eA({*iQnO`=>{b0ptxKPOLXWpR6^zMeeLM<}MyS21>FAqsSvsTfQ z*g)KK4lCg}IRj_{XWVYL2YQ1NdJXjLApGdBi(}8v5ld%UTQ(W)#YItzu>}K{i&~-v zL=misJ!i_Gnr4}6Ac?}EzaX=H{i3$7jb3jX+5leyt!V=;i%dC2%!4*nheAv$atJ4c zH7o-q#&X&a_6MVJYP_3*A`Rms-lO=-K2=WQS&r*k8G8 zb#I~X$jrwIpCaqlvhH{+;HgnjS!4=y9vnjxt3D?^Ue|!dW@0k-9DIQm+ zGd%=ehOX&34HqgdWV4WYMfI-<aBt`WtfcHT;R;S^Mj< z+$m%R6XEL=Zi3_#YsxB!E8D!`qKyQqYmF1SS+7<(8F$C<9;H*QXEEPJxrJ*&r~Ho( z4!i-Ne%f`e60N#}E8I&?a?4c_33Ymjt(^O==Kx z`%VF;gDQIFOp+K}uh+~;b-te7jzRj`fOO;|1)TscdrON+Cur1YE*xJW)dRenrLF7U zFx|6@^wgl_E`pS=L5>YMcBoTe&NRvdvL!2^2Q~FzWSHXS6r2sr6~pB)a~KoY7AolM zQaU?Q4v8XrPES z>s|#n6kuM%bVw59gd53mWit3`N+I7Gi7si=h|C%w5UP1YcpP7b*qGk)Gpx!(P$Vtk z81p3%+JkVv=4zEd=jm~*M&={aDqJGcWee10GiW^XNtoJnU@l>1B!9U=hJ@M5fE&-K zQfD(lHa=DkZQf!~a)Q9+?xTfi7>`|JpoQ%GnGR^GGpd-Hd2$AoxF+P4PA@Oh zQ^I_K>JM>F*-?&MMg|!hE)etwYOp9x!Ns4UJfxI(xG5!W{-r3&cZbigIr)q&(usw+ zJm0E+;7FU(F4xbGUYGjw<8^;@y^86-{a1MQ2XL$hq{HW5L4OF=pY7c3xzv;1PcO*I zy5829%D)~Sp26<13*DbxW6+cR)&wA&x!EtoLKd2Wiz33kY(1KkN!OVjVo03 z(!?7%;|$theV;IrDda8+`ZXH|_S*Uy5FyBp6B677zWI$*w|N-f}{ zza5^gn9?dl7S-2f5}d+23{Ow{gs^O3i59FOZbf8j~v36j`02X z-%oqKkSEx9~@glSROJB4reTRg}bIh(tEy*b4rHya+HPesuSM*`uuelBO}l8Bc{+b}{eKmT+_A+P+$4-5DP{DVN8HaoH0 z%xSE{JC$JGcYqHsP9Oa07Vh^bOp5s3pL9+JI)pTdgWq_>ZWTxBI~VfG2R_K#jQC$z z(%-$on(#r-cW*w8tL#r-{>e?qD>gun?;K<~9Zx^}C+Jw0uE;H~pH@avmMmO9H^#&-& zrLvW|FS{_%gU$49dwdD`?ShK;mO7ZQLihGZnD}iYVI75KVfK2Hy3P`1FtM^xZ6ETNg;YNi00aE2Pm z?K5vYf>gB3f)`duK%$J?hSEdcx$_ovU$8%X3Jmiu!lt1(YZ%dj`Hoe%7p=asPtg7E z74>O5nM^z>r_g6CbBST4o13?Kxsnita*a3~^s=KZ;NBoohH8uis zIez57Lz{5L?JMxq{Pm(KOG216*VRJoAVc+>rZ?J5Px{P!Z_&X|c{}%M)c_^c++)V- zsx4#&Y57MVU2Q6;$ghu#2poW8yP{QvrX?L~iJ=?%*6M(I3*!bXV-b1KU_yz^Bs6Pe zP1N*3Ld4`v8lcU8H%@PwO12wr@>+pE4Kk;eur|4)zElHdXxibUuN!}-1WoCOaDH5G zWU;U3GDyzD-y6g2vJ3iY`GKxsy_A($w;1L41;%)}?r742LvD(hhsO3D*vr;4VbQqW zSc@o}wt5Nh{B>EnoFnu{d+5YAvh;XHC1He`bv;W}&-r}0F8d!H@vi(hHY&jrvBb`q zX6)r$>pRL;ipLnPX0mIPH>+hgVe-8N=Vo0MrlN`9{8fKl4h`<8igCNQLA3>gjSDPX zyoEYLC6R=I-pq6@i#PbV+8IcK`d5U@R3QihJd7fYQx{rxRAa|ips7|(;$Tm*P z2Wb<3xpie5AMJr{>!NR2-Bu$BZ&O!>9T3fZs6_#gijr<~3LOWkVJ07)Z-Lyt>X?;5 z(O1)SwxSO#3&%GempznR`S%y%+#SkxxaxTBTXBAMNkjg(qizHxB#C&YE_qhV7BD=6 z8%dubyz7!iR07XIFPCzt5a^vS_LSniRNQg7XhVSbWNn$E;us0eAuQQa4qd;!T389X z2s%w$ph37}OY-Sl^|=~I%+yhe@;T^+gYbiyBaf%pmt2%`NYmt+={#g}w^GZw;iYrj_({gLDPq(RsZ5!^6_nlJ!vt?uE-mAG+QnsIrG;D0H2gKSJ%$%~I zaFOE@tu-7VzKv+2xL5Kf+!6CC({C0Wm+i-L+f3vVC>q=K1d2T-11fU|{8~&7biC?y zD`Um|$v&s*O{(5PH6!C`hE7=vn*K};KzI+i1(!ELvs*9MZO7epatyw_8v@D;R%Gtz zZM$BtTkvvoyy5+`00g68EWBO5VA7s$GW>!oEsQ*(!EQh*Dcc>yfN+REkFlUU-oKRyLG z+gOlGtryezGTbL%4=`=W#jRgF)u@r+rbMeWPiQS#kZ%cWka4|fAFfv>Q?%gxAaBuD zZOi-V93DF5KTE})!QG8w@)A*W40aw-I2hihkqO&zNB%H-QV7AShDu`|Lvf1*X^N$U z+ck1Rh=DcSktm^vUT$)qaD`*Sjp|wTaX&r{0VNt6QPp~Nm{W` z!x5seb05?QP~b&}M55LVHweU1{3M|Q1C-g(T2eB#of~G7o_2D|?2G*0rBR4^X zMUH)mS7>)#&_~O!G%XV(BG>p749bkyNGJw?v^8xv2``{6sr|>u@_C% z3q@g2<+Xm9W9V2$N57@?@}jgD#AlWzoKVQ~5Z0$0XeH|=_@qOWJb>U8*MLl?S+K+u zprPXW%h2j8|Ict`$E{wKHP_J=JM3TQX5xl9*luoO^J*{01br1Wbp*EWuIZYZ9A(Gv zu7FbP6>h)_&I%A7ungOT0ZS^=x-3;sUFUHx@ip!0bBz}<2oo; zjTK1$J8;=#vy_K92Z*Yfl4;daohQfxfbK3k8)hj3r_#rs;i9b5Ktqqiw6quFH!c8kg# zRpg<)6CDe_SV7~94O)hdFYkSMh^teAaQS%l+UZ&h;MwXn4|@=!L;GBQH&)}gcXmH2 zcX&O@yDi}4zsh5vyIJvJ*JS$oJp2vp`jZUpNvl}`xQ~bOpT&nCo;Myv`yM^_zp)#` zUpf|b$fXbZz%Q~%ZpQt2bnNllO!apwT~E0hxzvG(5rX&Cvo zjumKvT>Ke))R-6m!`(l)Dj%f{0w}(9O~pXKWgC44eLfy$96Xi2_G7}qmhn_-f7B}c z?s&ai~G~Ur*m4?bhr7&t*^P# z$PJx)9;M=0JjmuPNJ-zj5{#e!!`qWAiOc4T$F4zsOncC!>wk;@ar7$vAZR&I>H4NyxmI>o(9W z-ZwUsK*S1&@7`>pGOIieG{f-i=lSx1@(@3<@eD3XLKkNDSj`!vfY>InSIhE&bjwH~-Hc4kaaS z<8i;~erUVd9WNL%w}Qgc1sN0+mASw6H7Uv`Bdh8V|QJD{ntXYkII)PJaCczezfdereSSNH&-1W-1z)8q!mzzlTk{F2)(_7=pGZXxl7_QH{2%@ znN-D~k+n)FncQDnr7Q7?$Pr$Y(3D6yQ=;wWCICvP!|7R`-9hxK>|l)y5%v|5ucG#%>^+tSS!e&Up0Rj#bmmYnE3xe8~^QUoA7Fmr{=XU7z+HxsE z-0|P|I-S1cp}4uZ>Sb9NOuT+SsK5c(Yg!grT{aTJ%pzS!Gsd66-lPEF!L#B%UajbZ z+}xbCh6X@KNh|o~=JnZ6YQU~sY!pkHOplUpdJA|j;kxZJO!{dR4_d4%c5G!#{d**oWs?k=VYENno)-UhJsmY+oES#UMCmyG{AFmjQer2z=gMVknSeBuahW?AzS45MoRVBhm|WDHRJ-a$ zM5Vmsqg+GnQgU?vQ4?HUT{G9kLd~Iqz6CRlDG^mCTXWM;{SQ^Dpm)@AWzI?kvIBTX zXaQ&cIw$})loU_77&=$H#yPDqFjjR&_6IJ^ie6SUDyfQYZUOF0kx_#jLp2hTDV*#^ zrHa;_$?z96dr+vTJ7xbCy;-&kiV4i+U0oFWIf0rtl2lc+JpiuPd&g!A@D8B{nQ>QO!hvv2L&d&o~oa z#=^m-7tg8tm_&n|ag5`Kji>1<0O(n=i?rIOfl?$F(>m|o8-SnGlBfJ4Tgbx{N+_Df z)P1H+QcEa6L&80uQ4*=CN|`_GV*VH@FUr!CTx31I^;hMnEm=!uR_rL`0AgMaoD$)> z3{|EG+-syXFtoIeQhHl3Rb+ap!xC%2VNOSJXdi5#K|xP97gLGbSs5mven$9yc#1=0 zz&^tMR;V58tFATl5-%$M;I(AdBl!O1^*IO_$uW%Nac-~E>ophxDbkto^ty(e(|Qzs z7!?~r18Kv;SOF8Ps|LB(^$d$iGjAlPm2ApfOPdy3)MbGjH*M;P(iw~_xa*0!;q(AH z;*$;=M5Sp~%DnYgA^DyOTJ(J+oam`sL-T@aA)9yqA?nU+a$MGOeiyVtV}?_0KubXN zkUml6Eo@F^d?DLZn+k#)qAIwN2f@{-;LDjzI5 z7~4CX3<#uKaty#X$=XQgQ=_wg8cgcd&hg%Ym#fq{js4L_(F2f&w0Uim3 zb&;3@f+JWkdn|MOz{zL(dl~(<&>ov!MD|zoe9XpfXLS7=0=AAn|j_ z4$-1fT@EveoCqz_0=Qm21yv};q`TVGppv5PY^iNj_VPl^h^x7MZ_@R$uhNLg*%Gc> zRFvjQbt9XKn1C;Zwy{7l)7Xp?8?bFxJZUIn;FHU0QOH^$dGfgnzr@i&Ui3nW#Fm*l z_(jn;80b;YxBitxgHINsX=rjTOTtUYZ*l?TczKTcFf~q6D(CPyUbT>g#3dB5E)3F& zWkdUzB2(VPl~K^3#cDnjq(X6oK8gc;BNtC&7-?KozLs(-aQW6!LQdkN-Y_|!mI9a1 zoS+Z|F1KC=-!-%9Nq8}aDU5O-E>}(%&q`OML+9*VcH)ZMYU)Bts$NkDM-EO}5Ln7L zkpnQ3?>R)x6In?O0rqRv23c*^&_SzMfSNQF%M?Xn6gU4U;-5*l3N*>(9E;+eq?Qu$!*D&v`e$))>5TfTzh>p6E% zk3de@X^06k*H3LSXP#okY9zzQmED1&(>ifv%8Ks?2|q$w>q4b84|oLL&Bxg%)GUzkHaYsaQ+QUu>ElT~!>Bgf>2lD2#h4 z7o~O<^>~~hS;WcS5Zzhan4RcA8q-Cx;U?7ogx%7X31b3ubAm>AKeK(zl z-CKFe5JFfe)^cm%x*E*_=OHvgY@!5Eg`7S4d7=1RW}=+JAC;&9FSneKghRNJiD*%9 zn_i;MXf2YW^qezQp|vQ^V<~Y2E0KluZ}8c;+h$Qg2&K4+&PGoo_oJGiYMi|5QL|(yPnPTC z5TvaMpb3-j#)RwZB3@|J@W)$f`E>I23Xb8H1)s9@wM-oGrMght$vH?t z!Ghx#{OuhFg!|wQnMQUvuF-V4IW>?6>1i8BaiU&FGaH?Mu~tfr9A`Zvbn6=7UhD{V zUbHKix&jv(VV3I_8v|D2qNl8F(uryi2^kJomJGIHWs4%S^vAvW@1Up$^<-u#=tn5q z3dyJS7@ag*D=vn3DtwYnFRrHaoU*sc0brRIN6`cWU!%>)zOASw4LWvMcTYU1)0CD{ z%ih{l+=M>&K^g-_&5}UigLs~5fV?`IZRtR|##Sd7RcM-__*Ms4B5+{+nIfD-1rt(& zAk^U+7GD5e+_iA{O!4I{ZEDwDpl2y1e)vz|twS+gQgW*FO)U0PY-I!7Hnmx@V-+Pz z?A{$ztHde)$QoZ66SWDu*`op(he(8zYe(UNRyt2mG7c9zQd3bn#xA|Oa zn8j-?U5TB)Su?;xcTR4 zH|(E|@-Cw|9w%3gJSfYV#5GsPz2R$=H{Z#$yWIi)c$|DH^t(5ye9u-1@Z?op$?nq$ zFK~VxAb=xp;M7SRd8C-!ZGJPJU%^O*Ki|nx-$w4N7hgg6>?<2LxaGKif2;2enI3#4 z8R;+G77+xC>LRJJhUU-n_;49eW_&nayatP_XP8TUYz6->zEOn?B8pZEn3Ak z_Y;a3F%bR(`1nQ6KD>Qeb(z+9YoX`sQOmXW@;|&pOw_Q^53#$t`rY%hI0m=qQ*>fH z!(KVT#5tYLFDa)QN>lOMk8(*KOa-Agv87!{2PR*t2{e`E19hoBQ#3lE3@C^8+`Zbiu&0jSYbl&EIPC zNjd+-r&|=(%19p%N*QU3WKy5K3XVKPd>Yi?o@M^|65ZjtoA5nPW288{%dKbs@SbuH z!cFd5I#C?Q7CG0w$^be3x#F&muj#1U3hS3nD$z7J#Tt~|4jTw;y2Ws~xCMlQ>FKbE zRJ3C7x4`Eob8UE$XM5O0n$JXU>=G+xXGXptx78G(f1$qH{C0Sd$ByD;^KZ!$?yund zlSL{Y1}j%iVm*?!8>n=;dF7|D>@spatYbHi(R?wHUDPf~)DG-Vc-!T25&NKgMym)!bzh(ub(ZM3Gyb zl0%C>5&PE^{iFnI;z(0$dUJE0{MEurq(H$_LOg4ovT2iP-kNJ{ASWb_5IGZaM)|AG zOaE{Mt0`+vq9=rVF5;ye*$WOZf4u<9&HC>9<2i`tomE3bf+6tTre$XH|}<_0!Z6<*Y~Jd+#%Ole+7;lU5IN zZ&7ubVj~>p{xZ1gX(03mLd{(z(H13@?%iqSE#T)fjvXk`%C^n0Sl!%s2)ivB%kbZ6 z)&P>IlHo%fcY3|)%ZBEsOqtUZ3}gi{?@6rd>QUjwB@Pd#Q1{i(s(_h5%IW z0D1!`MKm-aQyH4N0Le_I>t1hRT3L9^YQEIPaHup*UAGH`3LzvScxlbtKUH7L+@_^B zoua@{WlWc;#4ADxP+6B5edR${c2|*?6sb?Ve5g^lx&{7@&j{I zk5nS9+qAw1$;O81IpXOVn7i7t>yIPUku>p5_9b#lPcMxIulBQ+@}D3uc}>oHoe z*z*&;2253^Y=b;HO(|7iDjR5c<)}4OD@YuguA~BY1P*##wqwq{MSw#ltmVwg?52C8 zO3wI_9&*;%D^Ul|&fTLA3DiU$gSKrKw1+|pklYnCV-vC&v{BVm`S!Fw-iPPF(ZRH` zY2ked&YMh`;Hs_|G+>sY6jiOnbzs0*Wve(rWV*{y#m?d`)GFW<=vc~UH!iESOH<)c zXILlHjYC?f;z^Wp(HYR;G0Zh9N}(8#&a8zbUePggPUy1$O<1N_XMt_Vo`y<6%BC)< z#895Z1aLPuO2Gx1h?+9865=L6b_H~w{JaG-Qc;1ECX=#`5VVYF$1F_mo&y$+Y8yVH zWmnv%3B{r`tzrIIFVI&M5S<|qT6VNhB7lkuE5d-SeO2j}2rrhKq0%B}ITLp)AEnD=hgE+OLp@QNgUXg}x(m2q#Sf=y0}{ z&ISUuS_@4@s+SvHr%U`LQ(9J58y_D-Ex+sxpl;=p-DPamL_9Cvde2COMhlr`)B{S~ zKaDFgWH8-J%d1T#tC*5_%LW9~IJO~L=6}=2aREN}28V)dBk{e>bPXQD3qw<@W=8n5 z1QkX4Wy#$*OWax*|{v`G;nUv1kHM2@elgH}qcUa@D~E|6xz! z^7Bs$g8VTmTPefLoY4i1j|ypBe*Vd^y#TKHWC$Wjcs>P~Pg{DjuB&EwVe206N&VTm6@8EW@y z;F*IbdD@KuX#DQYFqGf=?#*`f=i&f9AKjhhj1R0$7@|3FbAP#Y<1=`fv|&9U z<<=C{wzl0oS!{pT{3=Y_rFklQl@1u)G#xjTx9Lp#D$Lvk)E>Y?{~f>lO9o}%hDq$>EYRZ8kw6CVtwrHck%-sc*N!-dxQi1-fhF(<&E8@hV4+iYhOWt zvOHgJa5#oP3ZcmJ^opI@mkSJT4}El6gz^aLTFXfOLVVIltj*Zr5}!o3SWS{f^P+b~ zkk(-TEkWN(@j6*fJmH@K^(%!skyWLrGJQ=6KnVhIlSAi}U`MYt^zx#pf?tnB z0?^2uHS!D}*GF?qFA~&oIJ^kEyQbBXR#(*&!2~flz`=4W=Epf)gE&^(@k&{6qljP5!>|$} zLL#K3jVVp;H%`_gvzE^DyUJHHT}Rtj%#|Sn(quHxXCSdb^)CT@GpSBfH!@nx=H$;& zZ&Kl)V$PbhlAtL&bX?`qsv*HFCbnl%>`0B%l;}aD()Z&jkZQTy&h_2lAx|cwa?*9^aWNKGLK>4O6ARse0L(#QP)j?jWVlw<$JP)iFD8Q#( zWrgpSvf~p{83CEvwv{PZ>ykim6dG~0`mFc{w3)j)f2G&zf{*wGMFiNB8JX9J)frV? zgS=wqO*|M=LDi6{Z_=a?>d6~M4i*q*RmtJOdRuF7sVYk_kP}Yq52&h;HCfOZnLy%z z?la{Yk<_5d5N^cmb!BjK`WT2H^(BmWT&zA5>ru)c)z%pSk2ITKA}7KH1S>fw^rI7o zohXes3zilja9Ivp*0EU;iOJ%(C6f}Qg#u**uBQwTHtR<GGB2!pEjYSyPqa8j?s7gjT@P5`L9t;enyZiOe`@QZQGb`J6l8 z#kbLZ(5fiF!%2R=ziJzqJajd!FOnS&8CK?YNr6Fm3I^L_7Pqjb&Je}NE`Ql}3+ClG zXg_(x7O{D73z7{@qF~>kEqw`3f_>2RTxKytwSZ4LDu+v{h-77#nbMOOHhUx4a#4n~ zGMc*;WjE#85)hLHq|S*{evs#{Z05>@HBut5!soG+~U;yrC|f`Xiy7RyTX(INuM^qa1b)Kb<1hyJlD5Q=Oq z*Z$m(`={yZ=e%`Y{p&Z<GKp%R%H9<(DoJh}qy$(MF^$@5` z(wAC*^I+EK7-ZB9 zdFe$GxTx9~8q^QhTl!P$x7Gh{smBFm5xM1w>PtF^@;!_I zA?DxmSYT+J1M9r(eR-pcYZ`odK8^R=O>5+4g}-pV-rs&ZJP%^K`Roy8u3iM->HX_v zyQ#P0n|s4QUr#?kJVF9}2_O8w`|Laoo1+}(P+$oLWhcMh&A0tbc5wMwZ78zxjP-o+ zTTo=k+dZJ(WyXMJxb9C*vR9s@H{04&$q0RPFaUH*ob(S21~M#i9{eMt(ez^3Er+9& z2uR|=kBo=Ra(U0tqNy1L1`xRJTd zXd~|TAnznE=38x@u9p18+&1XH#>qdP{KfG2D5shVDt!eMTc)d(#dLZ*zV`w^qx^db z`7h{ZmI3(re;LrR`|IQDba3DEdcTKu_vVwYe*4WIG4J9q;>yD&7|M8$sIA#}CB)DX zDJw`b0n0wj%ru}WxA8SUU}=Hz%X!?tg?7sseYCO46!Zq z^Y+duhLZ2JNSs?R8FmaYtzbncN;BKnoX&+d2Uw~^nTqFr8snj?L7<<|EGvb*ENnO|XO~tAd>#Rc8mo-`>RETsnIrzfIfQ&&3(UM1h zhz$A=p2#LpLYrZ4Z4MKAj9?Vh6$qJAhf6IGb_S?*IQ^nn)+Ntu1=amFsWDxW6?m^n@lvG zI3eDuYl?<;X+?HGcExR-b1w!kAw!uXjf&KwY0#;&UW#(koWoGmb=@-o<|0WAx>>Sd z7)uV(j_|t_K&VTQqj4ie+T97J8r~-lGkYnz!`haH<#>{>qrYL>R)S-yY<2r&Fex%L zdEyI&tFll&C`{#Tj0L3`FGjs;n!cs2c#%1+I+Ww6VS0Ci{eq9M33E+E!ZtdkVbP0Y6A!Q!`~)anT}J?>tceUufz@cP#!<@JIV@S?WnXk z1_giorBBD8jZ(CrpQP2k$f=7>Rc3S;XZ`b{l?|a{koy!|o&wqEl3K>$73uW`T06uS z;?e-2Inyzl@LG;t_ytbtj>$L|l|az^koiwub7VI}!=mL@RsGUxQe}AD5XkFgi5;1B zaz=_#vtaXnb$AsEr9^sJahyWCqeL@wxUP_XnH%CUWK@CnlU|3sH!*GaquyEAUcO0B z3nyvYwc^**-2E+Q&NP#kc0J0bxX!~#rY6mb2Jm*Sl)_(cdf7lT%pfyf)zno_&Zs|` z0`OJ!Jt$J>oQ|!39Yj>t<1RtbIbmnFl|(c>1>KKBLJV8CmR0#~Zc?li-fu_Mt|`=D z{K4P+`g-0^FX!#1ihn5o_K&CI1q8eY^cNNBm46xE0jY7|O`TrQbWO;t@}fw!=7KQ;44>M*acIAnmgSsX>t6K~+!++I)P?)WktTpR#wUHVa(*}_}1|=-phB{ zO*@CW_@8(f7r)@S-xkv0 z@f)u!m-Bz)v$F0%Dhx{Y@4W2-#H@TLoKDb>19s~-JIZKn3OPLGMYaoYOSXCp>CD6_ zzkgnUToIK;lim3=jQiN4xWF#;VZZdHaQL5S#MR2UJSZ>!@f4`;X=E13Bb>+6ThKwd zn`jZ!c(?h+|9#n;ZPgh7qT}gReq%f*6tbPUYvwfz#PaYYXWape#nekatsDtxU(Z=J)MgYb1E(vYHtSs~ePx(pjaQ zUUgk$*t!#3%JmHq{NOwdhiVlS`O!Y3hN}{lQb0K;7OacHxWeYeO?GEsp_)tH(h7aE zE$VdmppD>!>k}(DHks3uumGQSrolzFByV7mGy7l$8V2AZXV@X;kNdVOIr)H)<3+Da z#F$~Z)IhFpdc#ytey{@3W>YuJ!4jG|`pWh8Q&8-W0&jveE2UVzcN>cuSP9Q5Pwtb~ zP*N!}D&Bz{v#Dh&XL>#MGZqC6+Pp&Dpm~ZG8bb_-L`NSb?S|_?R|?1yPG!J?MSe}< zAVa>{oKr!|6BpP|>RSa9=t+J6i3&ec)Z8wz0;{T`;E?&npWNO8LSZzsmMvfaf-?Hk zy7_sN21*zlsxC8zm}K0(xy)dS$KUl_hc7cJT<=W9L*so+wP-C%k)_aBBY{VzbUcYe z8zlT|n(+zLi~gjF5(Vqtp$3G%45F=*-F27Y*a|ZuG&GsU648p^W)xAW`(8{kf!BN* ztD(%|jOw|@F(~e5s0zq|^r0UsrzoB0K($;h zfC>htkKlX&eYvl+3yuYKw{r<$mO_%M^Pbct1-CDViLufYPK&&;{DK54s5F zV21KXcG=UONrxKIOkHz`c5TD*5Fy*4HgKFWTzB^oxkGC_2>a)TQn?vDD?6svNl?-tyCsFIVVj9*znH~#3f6plu|}Q z)4UHUe%BX8#*^NUrzd^hQZ@vRC1t}1mt|IxFQc~#h*=>=A@gs5lhzwpaUfX!A}L)0 zV`j;la3H^q&3qEtGo4M!J1%bmF-kgnl1du+I8&|CAKlt zxKfPg_bP8wBm2QohNiRva%Palpp9zo=|hu!5m&m*{wkdU&)xfMpZ%U&w{L!4#&6C=_xkK^=((xl&wqa#GOqE?%2fUA4lK* zKa`e6HXd>eP6g~dy-X5|g+%{H0lRn*qFN185ml-_x#1?nvSGe--`6!X?u=h`N8ZUB=VXSvDW0Vh?n<+pW~?_}Uq;Xz*X zu6K5i2=sLDCK&^zgsks6cX`-(Uip&?k^K-~U-7%iqg7y-gzq9Es zoW7Vo>~4GOSI5c->+1z%&geJ&q|y7Zk<~Yk#-C!G z{ZI0$GI}NH3)Xr2=k=5Xgj>s052&8ZJunpPHsj0d{`h|MTLb8vw_f58q_lTMw0Bc_ z4oUs8D*Mgr&;Q#VEpxwiza#fMo;LTGKhTrhVeqDh%h|8qCZ?; z?nkGA86RnjP2q_0bnN=NG&HkW^KJdb(U^_RhNF7Ba2JiuxV+BPUf{#Or1 zA@ka16zU@8R*O@~y30qOhC{bmpXu>VAU`?#8ZQ6wYttK;0+fO4<2>w#Z#;=**>sss zt*5!!tmw?g@B}e|14xl@az?!IY?F0GEot$qzVR(#h2eVW1%6ip2IWe;;iT8aaj417 zNxkR3hjX&F@MJfAXP`&M{C2M@gx5l;hjOe*8l`U~lIG&%Af(qwOhoUOW+K57SyGPB z<8FOj^z3l1Q{O!Zy?S~F9H9Ps*)*)TESL3aiPl0M0R;|=%8I+WzWi6EkVHgH3CvN+ z6nlWc{N+Ldwd-xy)0>C+;Z5MnQJ9vdr90PBNIfxbi}D7xDspnuDb$SiGi%tlcRqu= z-u6iOQ*GIbG6D(#ToD=$O>a{7O+!V_aW7Y!29zrlD8%LEt&qZI1brW_Rvi~9FH(e{ zFy#+F14+>0tC=b@&F6zqvE-UO?{Ybd1_BHeU|{)9dL8;y^a~)zP#(g@0gfNc+o4Bo z{aI2s$Qwa;aea^IS*S?~**?yAX)9a>Q7>x$SU7p!5Sk#As!zf2M!pMIc0{K{ZP0H< zQy|{Y8v*{IL{mI~olV^~J&(USooZo9LxERJ_0=d~&hQ@UjP*9990p8V7*gl531p<5 z9uhpQL*@7SU1zkcFY1C>H50VA>Hte~z97xZ$uqQVPkVV9$;w9Bv;@m7QJ>c2lG*AX zc@rnP#qDCLKdOdBm60ooQ9v+~u$KXhxs$j8$W#%Cjd>uEO86PqYeG4hn6#ijNV8UtX1+6Cu!W0{#P%N@FyB(!7! zS|)eWyL4&+@P$_F6SVuhL0bdk`@m9*c$BO(u7*m2L^ZQ$e@N3){6^j_7nz4iU4E z2PnCtCMm&y+GE3jkN56p1+}g&kUXZ%H>l6axh~EJ!CQ-@m|DsP%0$6|OR@!TH*ID6 z;rIJ6H8ZDU|nO;ug>uEfUN~jNcznaS)T-MJUR9&CpN zF>xQK#8MZd~vB_ z=h$cp?j(KMsWhVn_a;{ZBnf7<6Ni%b%=2>`BU^xefFg>+dbJrS&@&vV7eSFsv(HUC zHL!*;>zTgR$WSf|dDqaoZhF+W zFbB`^;Mk{w{+gU=IE|@g(Ph;lKGlJlid*i*g#cVYqrVA1Q?;3bn?*0Z*jS!fZ(YB4 z;7Bd&l4BHi$Oa3r8q-d-XaSTEMI41oDFKtLWr*>KqgJ;EJRt}ppsBr))2k>u#X*+Z3QL+az0;Q_c8D>t~Z&uQqPy+ zQJX-KJLqXgoKsP;nX#;7h{$2n{F;8+0vtgcI4`si3#d1jnW}*iYjz44I(ph`N})+y zvJcnGj8dszW&ubpvkL^Bl{z!5_|BH)n`bpk#L77hqQg@R$7CPt`<~0ikRcBwsbCx9 zj*Fz&)tDBsJ1ghHQQD6`s*!jkEm!f*l@JEWNv0K9Pyy6;t?Ub?Y<{We;{V%WniRhB z=Y_+pqV``i#bNbOLln@ZD2c&UaM`OGqoa zr{~7HS(U-T@EA@6X0HyAgowg+<6SZZs^zm^PrgHSP*uob`0)Ibje)IEBj}Um-ZoZdDbn|Y86CDrcKY;h4)~}I?l+2%i zcb0RFf&*uE9|6CUZze|bM_B>i@;%-i6~CTdTpIRpnQ_tW<`1L9-23;p^21)M&)w+v zD5M0OdizI+YF@U!paEK(I2~6l%{+$q4KYau42`w^F@n)nX9vFY=f~^*_;eU=KgS_` zkf{13*S|WS-`~oHbNjVimOBjBA7hiHqc4(=?pk}pry=UwV4wHj3?{#ov4&#g?;Mi_ z_K%GW>E?NOJ1S2rG#VXxEC*YzjN@+e?ewD?_`kclbA6;+dBf|;{r$rpTZ^_Dr+lqT zna0g;;$jRUk9zmyL;7zEE?f7|xi2RVeuP|P7@4gM%@1HAewcp#3HqwP`}wB>WLfu4 z_3`Z(S1qRU(a{x$J#sOHeo+r8l#EHFX->`CyVfNZ$(f$hfq<$F6`l4wW3= z+?*8im+N9dwY%e^YIRwtq;Q7t<_24iyKcL#;SfK?kzkYS3|WHtAkO58baON8_j(<4 zH%w)tmjNZV%;T7#A)uL+$dKqgZ*JVu^cJ=FWfU1Es4_|hcX77&fR z4I0=fIR2GhtIPglELn|2)`sPT^k7vJ9AcFCesDz-YGENSx-tl5Fs?n6+by{{7x){~ zW2$Q!OsazR+gio9kp1XBN+(da=(VK!?gb28OXyis?Mw(Rya}qj45e9S159O zUyhL=w(qz^B}u?mjRB{4F4>b73dq!BZMTT!$|rAF22X}66aW-`rh^2-9WG<7b3Z}Y z=haF!dZtSA8(^p*0GvY8qS0lB+?V`9h8e_Pwyo@*8BWopR7;M|Os|ZFW@=$`nrk!W zQ_0%NNTsFheHe|7i#lf|*Rn@BSVk676M|)L`htAxwBQI*usxCX`2>QFM~N&|mTi6t zr3W?gfFEREE1ec-MjAJfc?35E9TO`68A4GVrRG*;C;*JeC5iQpC42-fZ}6u@uqc?y zFQyQ@O$qf`e&IFpTgt)HbHmBgco;(}7H(}<(+&_*$7NBMP$SRm5TVBw6$eZ?w**c~ zn`x>_2#T!%WdqH{Rf&lK4>^@#v>@+3rIS{Q|XZU+}ST4VR&t zMTg*FK!sy~%b`+R>?TE)8B3g%x#=D%JIvqy;soSjZfKNjXPd*p@+R7lEk- zs#sYf-RA_oMcJWlkS5zs86|MtFa`O{b6q$c5^6q=NIFE(s)iB)QQZ_J-#rT#rkIOF zOr<*Jq zC0!*i)r&0)F=S>!*o8S#w9?*HKsdQ@ntLjpfVtZ<;p#$K(MllF6&zG2RTOn?3O-F2 zWxR3($>gKycQ2oQ7dmT=L*dy2iw(7?`KMwHCn~E*k*$pLOV=h-B%zf9nK_ZMPo6?U zEm(a%UeIGfx?5A&JEy>dRmDQQQ3q9qwc{p;q(%|Fo*DEwL0RE5(nPjRD3Jm44KwuM zRCowrcyGn2%}=vXs%9moDJojB3n^c3Lr251bk6i$-SgS&6B-f5>)S@$Oe!gU7utGd zn$;~U#;35m_y~2)2j>DUY*Ay-(%=N`Ves{r91Se!dfOt`(>{fE)3z{`VNmV$7|3fI zr;VADxOg}XN{DQl5~l}kR>8eAjcW*|a~)-@1L%&=i5b^$ENjahq$b1;sDd z^LPy-W1zEAP(Jz|2Y#I9#;G}%ePZ*E1G4{_5CrN=oPvPfug~C@b!e1wkpdtJo=}91 zwqC7s*!=D{e+1O>AODB{RhIMS-p3M(vEeArPhMa&|xG^@2At z1J9SuAYb)K0jk-b)@O?V+?RjxotXFu3EXZywYvMYKmPHL5rORFk)LIq$V|K2{F`_z zV}dsMpOp7c*i*H#6;C6hV7sJ!TJF4fA|-J3-JAV3_rrLcSSe*XR9+vRph2lL z&ba=~E7sxWcf&~*>S;?*A#DEDwZ(4SgY*0g9}`^Mj?@d^@c#X(Cp(4W*PY8gf#YVw zZhY7~kzCaNz3$&{mGSt=^LD=slM@Rp6QaK+;auX7n2?0GRpSxgqCkdE4)=^V64#a zS1-fPt-f!^*URWynB`_oknPkxMfZX8h*G*|KmEoZUy*%&JF%M$z*lQ1bEDdG_V#g5LWh+rm@ zS!9w!W@pWk^8h!4yMr_{c-i8yJazY5J@*fk%*!;TN9JKZq2f>KOXeQcs|tl3ji^k@ zbjkEmMH1HyfI^qMoO{lXG8huQ9^K6OJvP9?(2wc?DLZkLoDNq!X`^z;#hG zNoUaH{t1D_4#hL3w_6qiJBY7$l$VK|)Z3eqJ&>sUb0pE=H?K&LAUb6BUxSxDehwp4G?QXNFWTB56( zbI}b@_!Owb?T<~HnVciU_ZRGb&cd>DrpR?dd$Y_#?&m-Hz!))Cm5>~Z68lYNxl=z` zma6LpkOda%Sfq)?3PiRHIlXSgtSbqJX!WU2=`6~wXi{=0p-x_dXtJhHyp*r1p*)o! zqx5%ykQTL0dbdGMs>1MX7RA9j&#Y!5jm}v&M6?bg_mo)3{%X;2VQ+yP0Lr%ICfDhp zP~A#Sppv$VC<$Mo`+*bIbJuX$ny#cy#zpILE>A%zi)Cwa4!t%Si@`PZ0Cv=}ptM_; z@Q>6A%8ab$8d$PG94!BjieiH&tnENO$!&*=Y7?r)bm+!Cq9w?=4ZVuQVtN^LXw>NH z8HweuxhFX+Y&&^t>fpQ=Z{PPlOYGfnnQA=V`ZnZ7rY0ROE^u%LG7K#%Pxij$w?t}# z4*RH%C1-e6bX0DOYW$__2_0qBLQ=Mb93_JUW@ty5mmOpL;othp=##$wMsxRQX$l$S zk7mfr41M7=58lk(cV(7$jBi@h^^2~P)yd$IaJeuHvI^B^*;Qg}af|Et+RE1I?1huGS@pE`nIkgKzPViOVs(1;Yu_f1P zj&7Zq_?j7`e)^{|8WDq{&dS?yH?!#CUUcHd z)iKog5{`~b9C!-@$W1Djee&{`5amnoo?1Ydf0)cl|56k>Zi3U5LoAmu2MaC(gOARP z^6=gCx~aJ9Xo2oA;z|~b@97sa+@O_{`&IokUOND&Yt=npnIZQvmlFOHibxZR@EYx{?}<8Q%9Tg$ZMlVs+Sm9QGzS&4>H zU)wYQoiMjnOoasIm<0PwT$PgWyv~y?R=(vI-aWyRBcLqQ!40P~u3CPvCs&k1h`H$H zL1pH1TI4)!w4Gg6V9rwY5mG>wUuBs3$tHHyK>TWsBo}Hg(AKQkeV-8vvOE!m(SG}= zSJ8x;U=NQ<*P!rCkTqXJ|FHda(skC}Hehzw;k zrA7&j_P#e(&NFsnuj;8yS;_2VCrasT8_*NCbe6^C@(g0_Z30Gflvj!hgR;&H1AQZG zq^$F@F&WL7SR)0RpC->}m%D;iWd)caD5(-s`f4PLl4TH(tn%eq31DmPK)&vm%|UF( zpeN0)t4$%RC4r^AojvjlLg-T9q8fv0Dy$DaDh>4l@H)??E6BMmc z1uMRvy;DawTHQLY?W#l7A9(X)T(^xb+c(h)csk##YP{IT?{?DRT#3`QC`ZavLe*#3 zgCJR_nLF3D{A;uu*fYucyT(MyFIK+;z}KVXlGj{!xueIs4&Tn zg^H&yG5t|KeDXHYY_U|szT?p)G-p7QTJ8^HKiD_Rb5M$4}z;lnQ`vfQ7_Up#eEkCH);<4=0JpWOg!t?Vl zhXeMX%XXZe#?!?vvryZVJwm%xEl9=pklU|={b0WWi^MqjiQzxhvcfr@!UUf-my;WH z;0N5H;@7&pV{b(QR!wDR`|@|fir8y0)8QVXepT$)BM=&!oy=qVwdIhsQ|=yU9am*M zJxqgJF@Y|*N}>o_w9xvKINEkRU4vhhZsDySDYN8Rw%LMCxWoh%{S)IrOVTQhY7 zb0!%U2+QvTces^@V8XXhGiU+EV}_yi`pI(+qiL?>AY%5CWm=Ss+l?DtEvK+yIA5re zHV}GbeJ;Y*S)A%e+ZZ4+-!U!URi#v{z_>{R;>$3N+`GKa200e0HN^DECP9_@M!aIZoZbn8vqFM zk*VxV+T1YvS||(EV4Etb7OMxHGpXAu!8T>R05zx+>6cKikgFXZCrfatP5x3|CAeN{ zBj22w=wouNx{`Q;ekIhOEWy zx=IbO=+9t=vACL387Nm1uzoHKR)%TT7rLAk$iNez`H=dmvaGBU9e%zMT()Kza;E!* z)2#+Y*K#xX64v%E%=zl4>5!hnfqaDJP0{PKGR+ToKVrR7?RuKD&-BmMT&g8 z&MPi{X`G-YP!vqnUi7j|yo#7PmUQ72I2UQfcfQ;Uf;Jjaq)YU2RTw_KuSvN}bW6~H zMI`)S2k)2^dfB8<76wwmRIZZO_2l0fv=90Go@v-YwM+AydId`Cf;Fos#>{l3$qP_( zf7CH?R^nuHBTuXZoS{dTCR+?5k$oM-8*)0AHTS9k;_~_&yu7`59oi1MD%3C*8u$v3 zn-Ji}$Bt@}cTBK6cZq_-wxBXqR9mj)m~9h%zELM|-2eo3Jzm$OuMi9=2)r3*+6w@0 z7&95Vfo02M+?`@kxo!cejo=UHm3D)uxQX+aCjv{66wL~v(STG2}`##R5?W&raT8!Ui zIXIUiDkG2cJ~SugkN{42x~yf%^%AH!e%`rs1hw9h%g5rI^?@Gc$Oit6j;68)%L&69&pAY9Lc4qo~L;(TiW-bqqs` z@ON^KWlRqdGEGq6llX$>ey3N(o2(0hIn2ciHM34XnawH<^8Q(#vpVB#1cw9mrN|9= zOiNiTu*h{{mY{_S{Mm9;l_h~vmNewzp;jd`C%l>VL9MxyjWXjoG}D5M>tzysP>-Gs z-G-@5jRyhi>HID{iEJz-^8n42Of?f9u1)B{M9kK?rIScxHLm1=WVe$^!M)+Sg1k`; z2|=dUw;k8sq^OyBAjZw1tbo=OvWW=zvli2sGhPesc@VjDhG|ITamLN)GZbKw6OUTl zzsTyZ2XXv=SfTq?9GoJ>^fHjP82SB=FQ302 zWmEm|-w$Y(Jvi;yae+9%yc|Mhw4qgFZ1T#S&8f_AvMJPlXnK*R~C4 zs&9>+>+dJ9AgG;2MJpw|xH!fOCrqrk1We7-M1T1>fkf$iG0%a! zh(RS*HbOr6Hli{ZFeSFD9k5dNz5n&U{9j#Tb5csKyXSX6Y(+N;-dsTQB%ne&`6JAr-SkG^GXCYS^#MOyxeW10L+2ih!A>`;S=Ig1 z_rR2mGk2?_;%eN`=(Cqt-p6roX9?a8qz@Lhb{gOMJKeYq54(e1UCmMz-cB>SiOAp2 zng9{}O`Z^M=$}Ad*g=op$p;Gb_paFh|M+WQ4H@d^)cLc=1bAj!syiH;tdjdQIPzlg z0aK)&cJoyO6p)xL?WBl^3W^k;=-Dni(=`nG)}8X1c4ZeVNO9|O(dLDfZzDHD0T*cM z;ZmXgp&Xg1e(q5RetD$&J@-ML~4TkGH(Wqtts&df)!IPlr)`u{V3cbETR9{C!ySSv}YQjO# z%PcFraYd%CLq_2G4MV09;){s}x&=g}tsqpzg2BEKzHTedql7AN2QgZX(Y-!cms>>^ ziks&~CfNs?1OR`*3kpG@nqsv$^VhZ@N2C{x%ByV)9nZvqfw$)@px?yI1UL}g^|I`8 zs^NU>0$}FKkZU*{ILsG^L_sYHOOu-;t< zf`DmHeeE1PQwJq>MN>+X*=55t_qk{*Fo)y zxlL!bHnlV8a;Nmyg2}tXFkBCU$N0!hhrm4rwSo5&>glY!8#;oOk3eOPi(buJP)oAHz)aAc*CmztmdUZoptwHg#SZUYG&*S5}q`_UiKA z=`|xCgrN_<_3$3t0qi8}_b-7P+8 z#l3gSerGufDsFp|f21YF;yy%$V zX&2bMbNHPMXY)YEA~*738R$6k78RzeWCQtB^K^lx!H?*^sDXS5iZL`(s1_1n6f9YZ zAl(^ctsFjdle@6EiwsCb?v!7Ys*0@Qye(6kXL7V;RM^ysBO{(QOex|U*Tp+E41e#@ zAs^5L6}mMO9+W4XZV_wIQk&}m;;FYxDLR9n^BN#sPIkwO5M{2MHcg33^(P>E#2PxR z&LDFX7`?v&`v5&6=UOfcc3$7it@oWQME0O<{!c9c!X_-3&u6|5 zm;qoG>C9aunbfeVnw)v3B3(}?17s{<90+^S72U8NC$eV3m*JK?73fE13W+6O zym&=9JEV(>C6H+u!4v%IgNl2>+wF{lm*E#*h4$>g00VV(aX~3Oo%;_=3sg5XYKSxJ z!3JH~@F!mAj6+!L`WfGO;(gRS6Ls4FmzCJRZ5d9u3ycd@C1(!#IDyz775#<}6xF;g z+qPhK1C17at-e4doj>&yT(bWLyiP;HmBn9zy1qJozlx8Sss z)-#-mtP~kgu8w-JN}eb5W3*w~7wl!>GZ9tbwFmQ?q2p=6bY%elEy+&`xLR;(SBS_` zQ_It@=TMcRb2#o8Fvd16C3w3IfgU{rzKT1)BhCg$JTsFHQi`J*Ao)?ywsPHN-Zgp2 z0PJOrR+eN5)c`uEHnVJS)3&W40vl$-ZomOVyUYSfCGKibHk2-1Vg@rb?91uI90CDs zC&LUDde%YLs**FRyQYx)ahk~Y*w>Xr6V#@%tkFt@W*Fr-9`1HARRL2i6}>#~m*-1KsgqtqA7<#a3<4H=X1ffB#skm61Fni4^EB};MpUB=6r1HPTq z5%3i0@`W1qf97hUGn1_b2<+>*vrv97uxS$hVxihDlO`{i#KE^DWpYMljL@75K+X#1 zTgG$|2{xb%+9xIB+atpDvSj@*6nNZa*>6k&O7J-Jr;t7nqr6}q*qC)P18yq8u=*e_ zOE`3pQkF2~SF$RE>JS>T&*R}TY|(8lX91G$OLy23J=-RcTBJKI4TDGUl(Vl*;m>kloH+adx7W#VozQ^sVX!#vxhV#9=1)NnB zoNYG{Dv09~I`XyPth(3RmC0Y^^G~T{u>ZvutM6RZRhden;lh8tw{M5CrPbwFZTkJ- zxXp1DbkK4G+sl7ieV)Il!+pNE@mK#|-gPT~@VSW|LoRQw!_&~D5Zb?n;7#KP^08<O{^#ju8JPS=f(4#b*Vo z<<9cgpUbah-E8D5;PuT41xJbT0q?q<8^<(NI3tiCrs)NUB@4HnJ*R+-vmsO0AT zS^mPh0w=#(-BaNCzL_ph@{iwIl#zVz7Yz^R7j7p0cD%cjS^g)x4nMD6Bvyx7%CCL+ zMLynV-yKiA%&$TM(1dZI7k}+|=f9VgJ8q|o{6WHcG5KWuEk3~NK|W=EBLPJiuvu5U zD_6i4JV)DmxU2#W?)yNIU3Q+jtXIcr+6M}P&T-yvx1*B-pOnhN{RBTj?LZ|D7DR## z68Ump&3`!_Zx6RGWL0kEk8~H<^>48C9IX6zz%N}6*s%5xmw)WvMVqe9mesj`>yQVU zSrW5xI}3ZyIzX%DMKqX}>`S+>S_kyyYA+l8Nu~?-6*p1vB-h)KBA49WoRALIC5@vhN%%qzFv@pvaMZVsNZH8^>(mp$NZ9$-AtphLRz3;n9-1C)h6CKB}zj z;y7j8J+E)$OiYo;{OI;%EyQyY2ZCC6eD5@11ma^PI96>w*Al-cFy=E|m9Z%-L(bW< z7HDiHgS+2lmFv}v>FNHr|M7T1RZ0k7qy*??p}VW{6>BQA>JK=JtjjoW>`UBT z+g9X(uGd)gY9Nb(D}L+ssrQ5u;#tz;%_#S_tz{mvy82T7x-nJ9b)o2la%RZQvY1ya zKGaG?%R+&6^VaKm*40#P*NI_Gny%|ALy;x%M%a!9<(X#tS90O5W_LQzWsP=fK~`z(c0o8v3}Vk`j9Rs#RMQ> zk`QmuT5w%U44OnNcm@Tq+a>Bvh#H&)(y)oH5*$(g6gvW6>1#NSyx=btbGK7c8E6DF&zte8Kf6!l$t&-KUqq1{l!~yxYb$8`ptGSgb z#P{AtwivGZc@z#P5n3h-m_vSRWit=lP%`_$&Pt9NX2q1z4OfxaN-h2H14u|E>g28w zxNN$tVW3E$J}PT)4RFFV>dvpA08DR-x#X@FUDJ|dw>eI4^Z^T5HY|C1ISeN))PjyC z=cbwm4;YfKQBj#^QO_>(7HXxmNx8sMO~g54fZ^R`1}+$Y0J&RFQx4unZp8YB8eOVQ zZ6HN?z2C-lrxMw2L!aeJQyu&woI71=%xASJb8dRo_*iX9tk3I7<_<(_QOMz+aatLi z`1}2|joRy>8Oj7`$Kf{APzn^h6MV5FIBd17-dZQ(Bj5oKm?+Vh`mvSjZHk%-gbU_^ z7Pa$_$S~=R8l+g6(PJihE}C5Y4LZp0Wr}N-Rs4FuLurjX^rK#Hazk6~xPLrOiZVef zlkEI_1iqQ25t=NqN6uOsg?OZfLZ1=&rEp!m|8(WtPXt-*9kH~-k<`m&!x?lR8t7=r zhG5!6mF*agrviy@1{-JsEtp8yN^@50-8k$)Y;pb37VH}!M55==43SzireHYjqioK; zJSHaMUD4S#?)T$;bVy>W&zX1R&Vz`G0x)jc=VSW&q5wjh>X%C-B5IF`zVnM$ga`29 z->Arlp9S|e6}t=`K_roA!W}z$x^a1#hBNq)W9xw}S;<(`{&IiuubhdGUDFT6TxL+d zXvo=@Fsk29I<4jSY8daN4vI&&6lm&CAS&mgpx9I`1=km4@6IJ+&6(U}c>CBNPC=KU zGYkR^2})-INiT*1=(icNPg&P@YEgnL`*bi{iT_7qpPSZ7F*IRS^)_%@*M)- zMGaCUmNJI(q4$s>skkh6+V?ws&Z+^9lc6|=?aoIBd6l;yE1}sG2RPi%iYV{W(-SSsA=TT zGpE;Zq4;r@<*4d{p(`73>}|a-k{e44_eq`#;L+O@;lh&45f~SyG_y!p4?%%M8_kAC z#g{;)uA%_z)O#KZiC0QOHO9zvBL`qp(^iaQI6x|(;oVVj)+Jj#RtNe(8O_OA=Ax7- zo6~P(m0r!{D(yCkN|md5&8&ri`^!LPDNA|lwO*INA}2rW7_m6WzFV`>{gP2a1`<+t z*T?JJ%U}xY$bD=T1U3lxMzp$zh#CY^irfouxAV;;0DLSd5Pa&Lw zQO$!)!rajdEy0 z&_*BuYdD-`Qex$h8?X?Ku$z0qi4jEK{$AqYngrtZwttetM72ref!nH~ue4r!ahDu9 zS>_a6uSd^ml8g6~lhYXSNr?r(A9SuBgUX5=dsRaa#QS3y5|ihUVfi??B!g;Co^{dn zzXk8{+9i}+?Rd0maLQ+Y***FzznhMhPT_(^lbe_s-X-nf7=V-CI=kNt5w9d1^yr+25} zX?pAaP&}XWbeX@}{LN1Oqun6_Ifn=IiWv62Hc-OAf5KQlcVqo7G_rp`4o4sn#(No? zZ9IZ^0j@t9Iwe@>Q3_3i*%T2%i@XLPG2f@S; z!_hJxuZA=Jb^dgp|6m^$e>3b=>>~QcKYGJ2EOrGhg-#Rwrhx_H41$FJu=@RQIF9%#f0`~kxZBUa=`RwDZ*Eo}{%Jd1Fh9QS51<#3 z$$SyRZzub2uZE+=pG-0!lig`9yVX~SzekzOZiYR7X~S{L_%g|&2?8rY^8zEO9G)KX zakZ2D0NSFnTSM-x6;KjhC@iH9zZ}O6$PHvp?jMhXUC{D(H`vuyf1EB)aa9)4&aq6&E1px686!5ol*sAvN_cTgT_P& zyak1S>ou0CGswkzBU+-UijVWDVr8y~vjGtqIXk1<@2!Qn?^R2*M*UH-7T%l+vi??3CJ9)hf z84Jo-_I5KGJ+VPvu;Zr-)LE2G+puC}*sYzB#N5p0yIxy~sac#9dfj+AqI4{qc$VR~}3z2RJQa#idq1&RfK(>g}30`QUH4^sF zTf`LbNb@BM-#8HMbx4TwFSt|588qmErB-q8200t3T2R(u>pLw0r{MZs3{Ytxrb(y~ zcg#A2va1YbuKm%iCi0{NI_iDav?-B_&fkGOl4U-u46y=A8L>##Ajtz>1cz5n6F;$^=eVk%G$v@w3k6J z(Q>WcpVVrT=Q+n=a)K#ALt7M_{)Y=co(EkSwLD`&=vX~{XHqq`RrLCsSZ7Er>Wukl z-j1#-1bs;j(MpFPOl?s8rbGt^R3NLAJ8U)Z#f9=6u`+eh{?V@Ie%GS+b(8_!wKrF;!igF?cHS_|ytB zm{MxPb{ql*MUEn#i4laVYD+m7)0?9LF(5UP*j_KA*AkKOC=;SS8u>b-#v&AWi!B{O zqC!fC9zz^DeN)sxsB#1XF1=hqvOYb-+!kL!#GFaGQ13bRqldSkKu>L`&L8#`5KCfh zA$K)vI;P7&)2Cj}YsO24_}n}zA+{y2_Q$|g56z22 z*)p)NB8Anl3VW?{Or+zOpA6N;k3%WK4u6)6HHj z09jy?CmCbq>+eicv0s1IKFXp&%9XO|j8(4YwbtqK4njy{k4;q83HbGOl8$leMTVYgyXs(2{J`$$;_)i#x9& zr^NyVk2Y{sgS<%SNh65Y>zH>`?ps10Kp{cda-XJfCOPVX3APMHSjt#t>|<$Wy2!wT z$0=GQ69N%MR>_djFSzuLTG%DXBH`~8rj>Vqc~aR5gxR64*)>_l(gK6aA*hK=EqYd& znXDk674&GY&J*6Yd5J7_?# zzTyKCDDJxKQaPjD7Vs_T{8gNqmqS?DBV_J(H8JLgEVj6RS-~&>ILrD2woFZ{&!zah z1yTeQ{7ATX(sL(Ak8ex6(a(pV%m69|CjK6u#@+aS-fs}2*9l@{O{_Q@aYsu|H;f0! zqLJ&TYV~ymXG)zM16M<^dN~E0_eP6lk(7{i|JfHPe8<__sgVb!c(^-H` zfoaia0JuR|#l$YLmW{D81?10Zazc31LLSNB8p|)G;lS5e^IV4QpYfb93iyf`INuf zWO9CP1)j|$y!$w7+O+tgD-h9H@IO?*T>3 z$Um1ZQ45e{8fc!eHta%@CzHX=5||$iY5Zf?S(&jrq6^5w_$;I6Mvn5{o}1BnD%0-b{S*K!&Jy@6cDvu2HpJ`|*7em8tnctk?piOZU^2BtU`=RhHl zqwlSwu+hDg@A}7n$-<7aB2JcP=yyrz4UFM!!1)8k3T`AE)w_jBo-^7U_(dgpkVrue z5e{a=y$zpSc89Xz$noBJ0pR?{M`ShXped1w=UMR1@b+m8*YgxZL4;}t_dwZn1O~d5 zi0(YzU399*sAUoxQ0hRY)4hG=2ECPemjCv78Cm@GRqcO|I?LCt_620ZqOf-3Dz#Vj zPp!fN{5;ymSlr8c({E55zS~WkpIrgbxB0Gyw-4j(c-G1;j-&8vhs%DDZSb_R+8FM~ zhPRK?Y1C5e0rldaQ5!#4wg6X%+@A({33>LH`f$ER^?iT4lRGqe_@-^%vU=(dtFNZ}2d%2T{_xBD{s19>KS*4$`gS!?gN#+K%_A_Kf=YC_CD!oUa65mlNHGm?aOa06jkZ zS*DR4olw_)^B^BooD?0*2x@Q*2XKx(>f%-d;CIIlzie#qW>ow!IFqY{eKYudT)@UP@Shr{aSeJH&C=xWG8 z!C*jIP=@&YW^f(cD{+0?)LgxpN#h+qz{!ubvIYdojQplurI|9o=W%&-u#o&k6Rh0D z>}F-h@^EkK)_w|OQo5plzn>JyfY19}+kjS@Xa^)j7R%A0=ue(-!*LKyhhDzRsb78m zO~3lXgx-ch`3e3!94^rESbaOn#CPL;V~0^cs87Zr`pO<=xI@`JC23CQ?Bf^g$h9Bg=l@pvU`2Yhn(sL_wIa2*ac=3Drqozwkp zwfzkH*td2r5B9mC1h8E#9@~BY`Qd{g0e&(3loCV8e3AcT=_G#d4oQ8ro047)Uj_Oc zuGC<6AADRH&*{`(j@TOqF#X9G>}4q&Eiyz_!e-vRVE>g%6XVvumF?GZC;iI$wghU7 zg9~|CV+F5EO3TmH9QlCaOS1E_UfVC?EadFta74yNssEe0U{GjOBQM1J%M+@@vg+M+ z(1MfdU0kd&HdVk-z4h8|EY6}tKRuI=YAOxd0g!J^l$_jE@c=j_IXCB_0^W8Nu_D~t zeaIQW6{cV|#QCUz%PTMyrJn-MrV{U+!nT-Lz=+#L!>xs1Tn4oyWx!l@50~+=naeV& zutOn%9>8l=85~^&ZH<4iJ6_-fYbs!ndJ7_`X1`{6>f5!O$-TiD{hdp()(?= z44F8JNPON3C4PjQ^d;{-03hpf`j8uU-dRVlYq{q}JTI;PBjL&dsJ=Q`G2$vJVQQ90z1M~<>)IghCns`10LVG3@QcKQKF0mpetGFr?mrao| z%n}tmpLT?_ESG(J_zS7#x})7O;;iWMlF9A^=D2A=Y)W6;`WW88v@K;af}`e)Few-m za0T#bps>vZFr$ZI)Op5z3^A??M!QV-kp?GtN3aQqL#v>33Ks%l&el^$N46uxNtaPxU z(7}2b?pkK0)(MifC{6%7!DXK%{H=!8FZDP;bl~0>`(-CU3GDhQU7)Kb8MjC80=Wc` znXCzRd+gtj0a(GU=9(Z;gNl7`Y=$Oa4XCPPk{Mc*152~R}O^!goxl{#CQ6d~xumZ4XIgw)j*LnOnCA*g}x_QU~SE5 z)6j|}z7G_3xy{se$2t?Jhj93~xy*GT;Eof)n?ID*rj`hihg{cT32||{NdQ}wNW60; z^Psr|wW;A8u>4#E8!Mp+M&1Kps_t^unVP$9=bCo`tzquw|7kesfiIyp1vU~bxQaZ; ze~N^Mwjf$)vRqu^k7xA@Q2jH{ix`21r-6i1s&IM4Wp<`#^C6< z5wVbyw#jJ?xG2z*Y_uTjXC?vh|BoInQbL=C$qf6a={TM}zeR~O3}?6%6@%8F{K-#Ow@E%x-A^i@8(eRAyLKtc+Kn zJC`P4aSXT@RBICBJ#0NGNhWW>wEiNT92a&^l;9=krLj9^fix zNZuQ->IlENCPevtfAnt&9yN9eXKt*#LXQl?eq%1STsEdIr_{^>g9bc!mnR7|{a}_5 zxJy$hns#`2He>yBxhLEg>`BPYV9%fU04L)DWhpI3gAI+KVTx zY!hvT70sd7!J~{!-8IZ>wgd|RfK@PFCPonB^fOFI=b1GSF6T^9bd7|mA;3Y|v1??v z0iQ!n5=@W17OYK;UZ>Qt5=b~u9w6^KBKrVsK$5>RHYO!#xaZIiZzp5U7!7cOzKWyQ zqY;TP)ka+A1O(2}=!AzY3oPR6q5a9POsg4V$jh4`@#Vix3335)C9j|(MmFxQS_$Y@bujxNR>T$KP&V=; z3`t(%dgOqvdBy}_2`nsG8=XxNs>jalnw-F>MCt4bsBm*Yf1b8}ha|P7MQJgb_5VW!UbqMfsifW!Ng`671l)Ibh9kU#YA}v=Yqeg@*hoB#~0ADdB zBmTs8%kpIl7AFvENZd|P$jyUi=75SJhuk5B;Dgl+>Rkz;^99a4GiVkx1dJL&J5+{E z$gtYk5_cz7Eu?AM$mnlCB>j&oh|5o-Xn!YvFuj?jIF=F&H>hkYKpOnkI(ZI?@p?Re_ysZ_-teS^ z47<;Qob?S9N=mD&a0M>OzHqP%ixgTyL}GZuX~s8r-HYEzxaJ^0jz{9pm+^dt2uLlf zMecN^WN}&jPnUjNy*l=X-MiJl{-^)C`U4=b^0coX#F^a01B}0uXCLo}13tGOVog9> zu7j6%@P=+yFHfWV&+j&JAmShVmw35HWZlc)eqM|Fg2(wy{5Y@s;{jM%8L7`b9K$1e zzL3AkYJCO&gWnJ5@hsPO{iw1sc&dHCcIcOwe`WPK!eIPJgs}31fsgau5ky4x<3P}E zbt|7B78LP!*6C-lEb7pluBMo^{T|=$dg{#*9u~O*L{vog@Y%^tveiFW6^?JLVCKs1 zeNKYMKky;oVlMKR?}o$b!+*IStY*vW-R1C3K`u8IJ!Ua~-)dVQhDniBmvJw{?Jea` z>Bk5At-*US_BMKzHvq{^Vo}7)8`Hs)y3DJ?-WB7B|Bt~tOz6*?*O7R!)7fDLG1;ER z?ZrLs_0UE0CP4h6|7L+|;d5QT)Ne2MAO4#>|ChtA-yEm$Z49`BL{=|T?_r*TjT+2R>~oxkqFcbzO7k>;n5yNW{@mgXRJa)E0(PUF-Fe>*oL*sIIyF@d^(l z?m^2UCZNUGJX1Ii`h!xTjfe9{bj(4=5{#3B`!-QNmFeQHXIaN|+0#*(%$&$bMb+$( z3b3RYH5W%?ga9BxZN+&cJye_pXbzbwLU9TP@k&`GX2!)Ujb!44vX@}ztFsW#k<(Cf zh5)djs_ZB$>=%+Vsf&`CP(!sE=-Z;z?3cG-`E|MT@>MqV|t*BnYi( zqw7Ocdz-a2!>>TvqnAy=Yxb%DCp)QWnUMN2>Q!;RsnKh=h})yM3-FnLs;1gBpeW=` z&TE@Cbz__V6YJhxuTY7kKw*KeYGkHzPgRuAZ)CNUEDA9@aL%fj$O#&gAe(*D`@|n)Rt-aT zFrn=Ax`tQ|AFRh9M^b}|n}zR}QL~(!U>*6$7gL+E%b0N|BJ{wa#lWG^Fj?o|X1RXU zCq;+kkcVKZo^kFxlo!c3iyIfgYdLIkUcw8Ai5wT)-t40^X9DS7D)UTod@JMtC30@* z`N;(yYLd`p?vKhSR6#zB^UbzBdJGCLd1L5sj*t;m(IvL>kYE6}D}z4*!lgn*OagPX9c$hP-5a1~o3lty-Rc5t!X^1=!qJxqM zJ!M?7L=rwgkb*90mY!X9Eu-hTY)HZ*bKvc35AHN#&5>;yoJs-xO&L{@>~Fdx1%To&-of+>+g z2X1;3l=#gRX8m%(C4BN3xaQEjieCK^E`Ojne2_WAl> zVHz+8GQLL_Spo8KrZwA*5*q~8oWeFLlUul`0NW}!pW#<^K(Q)2RwYEDw;!Mz9$O-m zWd-d}mo74pF>^bEJM4Uk1&{}Ev=SfMKbo4!c3RQx0@4JBMxxo)CM=ynB_T1ZSl#xX z#zo!+Dj%smK(9knoVKAD&y^P)5*Z8v@AV}h6T+GfVK4G!ZAjP9B3@RNp(4)}k*L0z z%m0yQNdq4g=#e8brKocO4wE%nBuEk#;9%GkOxN{jbDPd0!^qN$pU`L+kSid-@0fNa zlw5;Z!Ik0ldKYvg0O`EGQ|77u5l2@PWm&X^C*44`i3=uiO}9aN1amS*02$LjIEt_o z3-px%2~9yu`>Hftmr$0WnC|W4u#F1QwcLyoI*WP+$QD(HD+{9fxPGF6(~?u~!Ek;!=v)NkU!L*PeGXy-4m)xg zBK097j!(f7wX+%|umg2VRZGxWY`jbc@)}Gq;X(5T?S&n+i=D7U)u0kjm9}h`_VIha z$rH8HWw(Waa%Vzm)+ zn#c=?o5?`J#!_fkwNIFtj0wc3EDs&tj+2mHrWIRPThPLYF%!_!D4A_!0Ynp&Nht)GN9=@J`|QDBW(eaB#Im(_N}bR3V_uT=lhHW7o)7r8GJuML)_1Q zImzT;pxHazzkBo`1lwblaP4OVhPcGyHZx3T7F$4OuVcbHo559!tgQh{O9zBUZzR4- z!rk1K7j96}X-L?zQbhbnE{B#N!m;{eX8)5?_d&nzo=QOVc0tU49=B*{Tceg$^45uW49D{{9_<(P zzcfEi;JDe3+wCrtu$N>vc71zP#c}XR42TRqIS_!%3piymjpNb#4-On)U-C^Yo!V?kD+v(Ta&kVovY&j}LNrBkw3X%n_m)yIr&p z=c)IH{&2zEmZ?|0SiL+bUgdJK69OPNNR?ra!tckA1b|7ue>^?^@K3vUSl1Sj<54WC zj4~WCOL~i*k=uVc_V@Di!_#LctW-NQen96uCeZZr>JeOgR)?iC#d5~4WaD@HoBZ87 z2VdB1E{~(t=y7n1@aI8}X5+WEnY!oB4!GmwaXC6n3LtkZKyKa`PO@L0+&*pLN9REv zd_4VXvJW}YGEUGf5C9T&Rn#3674YGabaZ5{*LEpeiyCyYl--bh(q08OFso6-m14QQ z1c{$0?^og|vIuC5LTId}DZRvuAn0oDl3{Ba@g0IIf~W z7_g#Qes+76TF53wk@Icu#XItv!Q1Qg*<*0q7Ob=M6}O|O*C-PSob~#h7xgI7Z*!fx zcBP-TY^p>^D3kNx)iwYIAwd6wmZl^q$_x{zgfb}9Ss1Re$fvcLOmB`ee0>nLkev0i zL(N6P6j>qEz8**i8VQytnXC?;v#4y4cs;o4v^f*Yyv+?sGuH*k9820OeZ{50D^R)t zCvQLixSwQ=WYS&O=@QR(C(<3S6?ZC*7@KfYdz&+H6j zD@b(SE(=F>Ek_p@;TAcKx<(S8YJE8A-SwgYG9-N;pKh1mD6@=yL>|}Qr7#5_fN^mM zRLv6WF*e(JO3nn)D@nzuSEav#>M4Eun1LZHQ;byVSy__`iZZkl?`>q}DGX|vr!eK7-jEX}QBur>OU)Gw9 z?APR0V72IFnO&6rjnOg{5?$ONxG4Tqs5{V`#6|y11~dg*(5rF-?wW|PEV5WZN}Fg? z^C@wVv9WgfLr|oF_5!Q;itn>?aGSCQ?{i}F^)Rp!xFY6l55S4CEVF`Ar6C7|d~TxM zWdU6(k+32cZezm%XHc|x!z5afK(*Wjz3{2ZzFwDsqPKLHOqt6NexgBB{F}Vxdb^Ki zd@p|nW{m4E5p~K#hqWR%#<9*gixvkd6M_4FeNf7r|C*BG}-1eG5 zb2exu=X_^KYa>%%Qj6|S{bL_WEqPNlrlIK1`6Ox~Q?lbGvZ()2OyvL2yY#b6xGW2C zU1`!@7hh{z_W9T@;EjqhLFPTHFI2LVaHIa*hvqoRsB_NcXw(9Hi46katdm;-*np`K4pI2#NFYc$2_5w!z|hqr?{38`i% zgzzgzqAMhru)@xtCl2E2%&%j1np(R@)Mwa(0PB z0}ec96197$8pw$8u@ta<2BM`MuRNQ3bQU>uJ(yZ*dUH5?6c(zsHG5qFkAjkH$B_TH z+6R%FL``vq-^pl!!j>U^zS3IO855_-69%1y4l1xz1P`L45W8y8DngFBZyrF=vTkS> ziK1UB6hVVQ4)=`9{#Sk4wvi~28u7C$0~U*g6W{30kpYrsSW(KmSKdfY zUX~Y_X~{AP_*&3D(9eXLCeX|svsPS!21eO+rey)fSv162B=mo5oPYEt@R%v4eACA! zGGMG*C&%s^HGrH;A|bIG-u8hyLfnits}g6)aq72kd~ddd2Sv`4DYStgH&}u>GU@;7 z@&+wx#WJ^^1{emihe#|;shhH4 zzJV0G`Y-McWFYk6)DH;->>rUcTm3L@9w2ga@fA*2;(cyK?O&|EvxO&pZhhh0V~w7#A0`|Wi6O{h*@m*2kX zKm6b455GMBR=lxC)VyU1$33Sw7hRpr78iT*5QnqR^P%S zJKSG}vztX9{%LEaEXMvWjEKUIR)10j-RIx*7a5<;%`9W_IGwD>yb61)z@=}!>Zz}J<>112yq6^I+~=5!O4axM2iuG}Bx zldq8S-Q8$kz3sw&H5~6h{I}!a6dK&R(g}(7yD;Sz{q6@HCp#4O`H!nVOsC=T!766R zm>yrOzJWt}bM;O$vg1qC*!r8*Z^OV^!I9m7CgyR#^nAF1?t?t!m;DpUw12p4-=R>C zIUByhR{&((U3RN4hkhGPz}LMz;}ibWj(LI*T??-yVdVyGqgL%1XRUL zWdQnx$LHsv$L8f`l-pMC$IbFLEAwLY5>Iuv+D`9>!{RY*EnhcVzgNp%ay~ot5#85h z9}M&k3a&vVNeBO%@eGj6#i26(_R6q%VdWwz`UrF1+2ln+Fm7^_O)bO^O_Y>2+C7Un zs!bFXvHMs9xG;61GOI1Z;<(81$w1v)>OM{&L0 zj=s0NGaM^!FjMq;g_IgaB-%3Gm0iPh+jDt62XQNVi}Q7Hh&Z6}Fg;w)8djV1-v^9R ztf2jX32v`L%rB=v*FGN3Xtk+ICWJZS=y2B|(wa~vm6;Tp(HpKTN1EtjA}`78wLY4* z5)~b*6R+39&Xu+$+!Ck!qw?-BC{YRGuRN*#Ld!Ho=_KFl!rSR<~Fey4;oX-{aZsJRw!_GKs*!u z*iZh*aW+V@GWYv>{iuN&`mp3SiDJar|AJ-f=hTbV8ahy_w-h*UeJuhVlv$$X%(_0# zLY#lYjOYvB9ujgL&1vrfs!5hHu0P87cYu?!h@6I+E!b#ls(D0NQTg0CH%04#tmA39O!_w|LMWz2RmYOYi(G~*JkVluxp9a@S!>V)@aZ}SOU|ms++o?9kbSLd z1NEYW2jHi7U3D_ODO(XL>UD*h-u1W!ap1ZFR|{(rgo1qyXcW0Px<|;xk{qaAFr!T#z*6vZg4gzgp;46*=IBu){F%~{HNDTY-5z2j!HUJQ`PLG zoFZk1ri7!&LNv!f%UFS;gDP(Db`|*`AF@diSIFV0kc6si7zekg>wuEme5yiH!RO<2 zl%fqvlv0!aXn;i&NiB^?+@UBlcD7=ak(?=A!-Ve|0XmbsVX_89W9}+uRU*TFJcrB7 zaxfDWh_l5Ev~kT6u!@W}vN!!s6>2KmO~R!LUqu81q{JOu7WEDO;;*t=^8(M%)J?%V z3}Q!jU1npJIxOyWvj@9p$B@zm_yo-Va=J3OWUhP5q&8(khsvT}R)XF(K`j}9FTg&@ zt=2A7)E_ZzQ#C0Qzs3^GAe3zOOE${rZ2!Na# zDQYT@7ZsnX$yq8aa8pTO+m&3B4~QALp245Le#N2J27EcVm??EpYlG&3>`4h>lbxqd z5+}8j*GT%o;j+Cj8)m8mJ{xL~_?7tL6;a z27R$K2sZhsXaR}uGTwC-Fn~t<$eNs3c;m-OH5B*NAB@ApDs?gi)us6u%lkv6|hz%e(#S$FiH-T2rnN;v?=%9O;#CuVL63Uhw*L!$W zE3J3?L@oxYjGMHfj zGeB96F$B;Jkocz82tcBbnx}CFCMqiH91o*Ye?;J8M4q@ za<>B(QT`kvFocJR%h}*ZF1gvjPAxSkWp2@Kvg9DBO~Ke>RA1cXDr-QvaJ`;GTwyS~ zIT*8ToN7=e_*G*04YWjX=j~juH6>^gXvRz)j0IeD!&DgCusmeiLQ5ds*D)&~$XQ%> zEG9AqNU~T|@*tE3n?u_Rbt>2>)oXqLO496kV* zUCU&N){LJnTdxHGP78L8eGN>FGGMtV2wSi3-1WS%TD{j}9Yf7#2Qf1HMM)IknBfQJLvDz|SIw@P0TUS=0+v|biqJ8seX z4`jup$clc08}5cJy0SO@<9P0O$=_IAWIx+E%s>-2om=!le>o$ezB*mxk7RUSO>!dj zmz&k=>3A3bR=Qbz(}U+1Hhas#HlDQh{^cxeux$T#&-zBk!p=1NbeG$oK*WFv zK`PUEs1ZQRSzsfR)wZ#g`dwwZzDL4O0Z2BizM!Wt3K{@=F zcW!Jhk21~0kGJpi`{^(ptyG1wU%ZIO&6G(5YeA`l*vpPuw>HYw6h}lT4oFbv6gx&Ji8_xi4vcGM*{&>8p4N*UXVc-+BsjySs-XhB!&9zuvf@XCX*ne%#(bmqY^Qo51hf@cE4~2 z5+^45_%&Jq7^IOvlVYArobTzxbxw55>jCRbYKYJ#v?@H>Bd<~XPY#ARNc2+4_fkZO zk;OtOj5)v1cBUw}q$ab#U9K8{C}=(508;}b29gSnK8|h6&Lm{rXVQYQ0huSKb(Qy_ zm5r~UO~v}o{(8ys2D97djLC^~R7`TIj*Xh2 zrJeRi(K@?}}kYf0F;Lme`ao*R4*P~FN? zFz@|?2hY?pKZ*oFaHJY3?nT`(4N)Y53K^j;!JlY1UVVZ(lWds`Pl zRk!Q1&A4dbI^pytXfGwGV(&d^X9Gc3%74gIA3ddnjC+$Xxa_=Y0~(?d2D(;iG%HhH zp5s*QjSr(kvP7#Rw|g$A`&+Sc?TD_JEiVJ_9fR*Rx4 z$gP{}_$^+6lBI#lylT3pqnzD>dbdKEon?`SS<#{4Hf*`B>#c(j=u$941QF}ysbWVb2gx>PhgObyDjGSJNF^mMv_QY0Wol#`(*fH8m>}!QIEO_oP)}#N zFkM!1mLy4v_P$y#$6Cews=ZBqI18EjO-5sZ{m)+)uaB!|HtVAnO=2)Lob&*W0^nD2 zP-l!VK;JBK9z$CJ@~D~XC}bqms1Tooo%udsQRH1)dK)=UT*ESG+Z!&}42S!m4=+og z;cn9fn^I9{RYKT#Zxc+;LgrSJ2w-o0jSXlI*;9xm#dVtNS_0lMXG(%Ox#+CWZc12% zAlxkjX0n0`ad%_Hags|UYY349;z4&EBn~ZI*HACml=21n42r>t&KD&&1T9yEdqNbHy~kJzoJaR8MekK25k zTpUXV*vi4Aj;UPFx{A<$$7vrLebIDN)55tNZ#_?hocrJryLK)jGCVo{z-p9eJ_-hP zLWOK0m@I)n*Rryt{KnQZsDLSlSo?sf*?PEu3`ScpN-H90%4;x#6VrCUsbDN9uLQu5 zyI#pL$vs4y{?0>8T8pcrwH;BOfWD3i^=<0MPYSsM3lP{8oM%=6KLnHRKlTT|$>gB= zq-`p0ViAjy5wch0M0trWesYVJ{3<{pC9t?-piqHm5V282$2I6CTZWYRA(O+nVOgQm z7-lfk$QgVm!B-y`Rmy@HAiVeoIJ*e*W<8gHxG^bK&1DrnH%;9#VBulVXAtkGVj$KP zKts;*Qry6Qlg}UUouVag&{gQH1&XQSl2f;chm&J>GOYBnn>LZU0C*O;CGN4;juk*2 z#b;)YkOi9w5^q#(u5G`us7tQh3z54#Sy z#?uCj1AfTF|H!#9>UO-{dEpLFZ!r&zY#tYn>@#i+^C9O|DP%u2j7c#HD1{D$yfK|U z3lurvL^PS1p`AP+jRw_Wu3Tp^Ws)Ri0L1wB%?VjWXyf`cp z=rinIqd`sxc`G^u#657*a}wEB1?$X2%^DeSu1r^THysC0!3I{8j?pwI$LsY<4rX%4 zV)En+vWyLPR?UJp=w)mx?#Q?wz!m09WH*wvP8kJHvKwB>NL-X~XW&ja19AQkEw|D2 zzXiym=z%=DMi02EX}}O{rv3hMfVKpo0`A6pi01^n#OujcHEvcf4_nl=zZ$plFOIAP zq6n6opx=b+-}bv>e|d`vI%1pE>s}_Z#fZrNxC3FuKOjrB`iK6w`ub}Np!sp!PfmIQ zn1ZKibo>SumGHfsU8^5uS^z8ZYB;~~;FdR*D&o8Z6-;*ycrt>Y?TnKA@b(c6ZlEDa z;0S(uJOE+v`vEYHH@ytjkMg*~PVTh%*-{c&NAUpsZ;N@=kX@c6u zBXr{&Bf||eDI~&O{l^n(-rrjq3CAk(W;`5RS>AmgQ?C9v-CFR0oJJ_6&%6bIOQ0w3 zXNYo0F5MB4kgbMJV<8eC38Fm^hV-5;zGQf|!3tq=z zC35{^(2szH0yYlB^5ZZbWZL~`@9%~sx(jaqc%0x+0=979qm#yypV+TpaY6O~y&N32 z>@=W>=7d@AaZLT2-?&RRt3g8J)$m;ZZPOndBr5!bodZ_Vp_d4G&KAXq1v4-YXjlAULEMrr=`#VW!n*~7$}yB+Q88$W@57!Lgi zXwPO?$!b#|)MhJ5ymx^glLg2S!Apsfox>6!hsBQ90Xw3*UO^L# zoXk+=w=AOxEX8fF)E{M);9@xS4nu&JjgE?gevvs7iBh;Le!V{H%NewP(+BlA1NoCtVdv7qOL=4&Zop~f$67MV6HfJ zG7A%qrk>=^1m#l}IJm+pbfrO#kb(|-Ry&KVOSJBysZbd@#0QsY%V-BJ3!+JbWI<}4#S87B0rd_1B8M#;Lj*(kSel8On?DdXxh*Q>z+D;9K@KEh{(0vC*g8N$?Ns{$@7&qrpcJY z!+E7@P}JOUB8Qy2ifVPDTXyv17HxuL1g?W#mjBC*J8&EIdMzXL$##XhLpLtJpmiD4 z0*dV19O9WCS_uUWW!__?kbK3?14^wj;J>I^I_YMx zTm?|T^`pB0!d-`4GdEyueT}?A9)v+DGbRK%9i}_8v4k^_!?7lJBXpu8lb&j2ydS)r zhrAt^;2%e?)qxgKrZNc>LZIKuxYa#|Qk88k{t(wjLWy5apN^cF1rt!&n5<%c4o#1W zQw(TXavM^E(j=697x`1jK-Zi`>UW`lJ(^;^J zZiH{&rk8lYKZAUa#4bHojtlVU5lV2Gg>i#p|mIBYI+MDr8&%h`~=#6bY_R|5YRggCNuOdJK4RaSa+SPwxFQ z$nUouxeg((UF2n3QmJ&QW0U7KbN;Q@3Ko#d6;~eO@^=9>4yJHh0ETi?X3RJ)yi&K%<5NkOjmBjtEVRS*^&b@{q3yW!m9% zngVF0;PWh079phrdOuq45ppR9%_;5Vhv^mypjrv<3Py58Y$IKf0xB*g>EsqTaHt75 z=DW$;t}9uo(C)L|Ko%=g+755ebGy3b5iBW`ia~hd#z>?}C7&5HCZ|qQvKxO26<0ai z*bCT>kGpB_g>D-7YS$+Bqh5q$2>EftTfZF9FU^vk!&|+GG7<%IcjMhTU`dJuchQ?5 z(IbahOYvB&gaxU~gj`}=d&_aevoL~&TRG$d8+yiZw*vq<&;sOWgJI#$1%5O)27CRr zjV*~d1E_VR+S?XLBWQmyEm`Ph$x*T7Sl{HCzEB1^uY^ix(0aiqFoTv0r z0wEQA*Ooh%292$ZMUNl9?asFKAms!{tc8PaB!Db{dU@_YWZEbLrvlP`81VF6hHJI3I zR<=R%y5>(ft1lYycdLvd%KPEqK~qNJMMDFA-KS$RxF-q5pG8}s5&xU~N#!_0*I~w- zTRtlD%0!l6lAOt5dx=3>@;fc#FaqC&f!uOQ@Vlh|i_Gb=;B<(sJgG%3^NfkXL2K>} zvW)wR2)PCo*n$Z{uW*O51?M%3d1ChqfSgv2)sE>|Vgx;iZ^Tg~WziX@pMgc2)A>8! z76QGQn)i8&JWvDK6f*ALF@>H4R`^LGrIf~%2kexUDLWSR+}%xwlOj2g29oK|GOM18 z1_Xg^PGG#v<%mk0NA-YLm9-ZwNWoh)U6n|#W}37V1DwR%$`-W($=TQ#rm~F$LLrn? zG@Sbh?KoM9EG~CK%IexZ1%ap%f&mGtmqvkIg&c8=xs1X1sBbg{6L$jk;SBN?d`1b3 zwQI?@s39<=)Al)YC1;Q+lrKo}koYfQ#^=8yM*n_OnM?F$AD~ZOv=Rq?QI!oYrN)&yu`n=w5hLlhx(Pm`y*Q1aAeTwRT zn*WO>QitZrxO2DwYcc{}@fWKxEho6+wb-BRB*ZW7`xANx0I2~5I*>{7zwceZ?W9ZO zza58-?PYP?Hdeq}@30@X@BZ@4{r7(50bVQ37p*)B*@0ZHF%tnY^Qu2i@)*zU`yVDe zCU7IuX*fobHx>Z(s((ZlX;^)8+3nsbSIhVM`By*4fPtz7k24SU>**qoJgOSiYuS{B zBe=LmWpj~-1*Jw1r@>>!l2fvyyjZ;gsYjE|%tBkw&eIi_6G1WaQLi#gb(}t z$Ns~AKY-@s!!Oh7%V8%Ti-RpWqLFW*5_JIeOK;IC7XNd4vHEJ5j`yIG8Q%}*&rX1K z?S}(6nD!%(udBjx#XMj@WMuR*xRxxa-H*%k5%Lo}fb9D+>c=aWMFSEr;~i<-yX-GW#nbe*&p&U zML+aU!|uiEMH3n7Y@p$WakXN@dD2_kMp^x0aDr#=T-gLK<&lzdT>CArB@1CRhdp-{iLU%g9 z+YJ$K)jaM_C+YmqpFQ~)?SBiR9jr$83=%75r{4fXhJT6{s3lPX7;aYeI{hs`q>Y`~ z&n&Y|-6a^7Zv^Zs*QtFB<7yRx$Y-Aw{TWlt4nQIr5NUAqWgx2Ld--JI}rK@_g%1uG~@w0<9w zzraZ$Z9zDA*@fKY0vy)lbnE7-CS@YWyk2Vq9Z*PFf?mLjlz_H|*$CI`i(;5eBklq@ zmpDM&<)Z7T7!ZNPOrFUo6CS2lS|9i|)CIBxX#k?sldz;!&rPezL&ZIRZlSjsdx30=T$eDOl zy-5kt6b6ngkTo588a`Du=p>Y5T3P=()&#V%tZ6=mLs^XyDO*ZX+7Dn*29tC4L zq7!mz-YalMb)Iwm;&#-F=!r_83lw6}LU%b&TzAK++T>N!(ggBp$fg7HH+|Uk`l$G> z4pcQ%IgX zL~5gq@(@i~qN9oyTA)8_eQK*`uK=iI7AVU@@DMIOBpkVO6zkVp0Nx@yJY zNGMBt7Hcsmb53*28)V?9>8eEAa*%qfi^^mzRZ3&LX`Q40>7zWfaZYXC6vF> z-zak^0T1@1K9FN?$iKjTVXs&AJs5XuH~K&m(^cT*=BB@QUUx@kV3S}+@KX?lvL;at zh_QKc@_iJeQPlvPa(b%i9OSgBa{0%EHUp~o`V&aiP>w`)94*#WWs?wLl!5fJKH}{0 zwi@X}R0VP)G=CQfSAgofhO()#K$*BfhKU&GrC1@&#SC0mkeCX6j|?{PG=eaY72S|y zxhU#Z=m<45xnw4|;<6plf=V6%EQ#D^XDBg`ndGj^%7eVk~wE# zNxN-01N4BhRFY>pL5eQt-7!`NE5#KOUpax7n{i6Ej86~Y!)9-U9!*N+B59=tB|z}0 zS?nfZ38DfzZ({mMlXPHXJvX*_$fBX36E4Sv02Xkn!ZEOqM;u8h!oyyZV?ffLm(8Wp~VqBuhJMoqo5Vb7X%h3i_ zo@@6hs8s<9$29y%(fgXdgdjOHX1HXpw`eowft!WO09LJQaCCE^H+9yrc2UHmVoTi&NuH4=^4Ym_|Kvd=b(0yEwuP*)Uau3a<#v!}sUIK{ zshF!-3|vQXAmKL6$uMUppWM)1EMLqsoX9h=I$umXQb};bl&7&gM_vM;&PVl34P=w?Vr69*P!jqv93_8|VCJVA1V+z?oxQ6E=1^9~v zD& z{Wj52KpCI}f;rZ9l9_oHhCAS<)72FS`m=2Xj2dSmWk z5n%~;H*a%yAfs2a?F~2N2XMzuF1VUMz@N;Q@E+BxYHEHI=&ssu$dM8$se;}xw}(FN z%H7?lgUT{jLK6*Y)n_DfuY z_Ry9&)lq-Zad{p5Rf3v@nx&x7qIIeIDp^{|M~mUxeSaG0eB?Cd0Laz&)C$6*G@rd7 zB6x)gE*ONh?5>loj@y3aICBb49X2RJMA%fEN4``ME%rc-sH=($ZHnwntuR5rX|=5R zhq|d#uIXeFFzfRDqzLu2t!v==iOr+mE|?dzWKn|Km5tTL@ZIp9Cn1P4D4Wl;FnE?4 z`GQ<=nLXu}ctT)0qeEmki(}{Hm5_wt=BO3x8|0$cUkSCsWWFJ@L95uf93D*4#h%DS z8O|g_XL4L?&LK`(2~{Xa-3=cU2!bB)k^{tr6I(3gcewvqLWq7LiAqsR>R2Fc>AR5i zljjj^XMNG#w8+Y?PU!jr3ehHM748baa~ zJ;{4~km>wnpLM~(7@&nsLjd2hC^>yM_s645#W8?h{KpF@s&V^wm*kJrIe9JL0azwW z_~KV!SGX0}m7d$^`#nczAv{oSy#(7Mp*($XBuy zPsZ#p^fPFyjP=`5&gq@E(P?MCw!9qkJuQEc`Q&EKgbY-;-s}Fu&|e_Ml)M22IdpGM z78hrpZFO7r-{0Zeyqhl2KEnV1%Q#$qeg0aG>itGy(?yfb0Fi9g73xpP&mc@PxUeU= z3)yS#F4$+>TsyfhYPUX_QHuF_b2;O(uO>|Hzlh~+^3&7ZL9X&@kQV@^tU;a+|20n^ z?eG6l&ht_3e)mC^_59c1P!vUxgPyrzq^3oosZ&31?pxmni>_+h*G##{u^*elACUpxsF#&Nzr|XhCv;X(oi<3m!bL=Y3dH4FvxH- z)8(W%6gj8GD~LtZW?16B2u-4gEfkeJeuU(A9ux$Fo^MJQZzdfC08v1$zbhUGNjQ$I zK~Xeigl}YHNAoU?$q}2~$y#K%y+~>3PAPZ*CXdhqE+j=+)#ec$O@|CY&Vz~P=V5e` zPmQ>-oGdpB)mpTmAdG5XW@x1;W7=SLu&N`vz6&%a?;?;CSJ*S-8JC9T)flByKx<@D zQd&@#s#;W4P8rw*SCc`@iPBoyVV)tF%ea^L=0*-{9dA-@@3k5o7?q3D)CrLtM$)Ql z=?&f7Y@F5<()JWhesH68*Rtf7-2O)=W`*)AhY$oFp&%b8WgLzP>Kuwj zqSTBzPCa0xLz^GJKh5=bpoFFBhWL7RRH7OQ8fo_ITn|u}qIsGE0R_kzv`Pg9>+7ZC zOI~QQEF&O$$7Cc?I`hFSEEz2njw$yMfMIIXTbTg(=;W8lOt21{1EzFco70>QYnn|5 zH9uC_QKhJ^1gwO9{ez?Vkv*g1X&(S*AYj4z*Vc_wlR9V0e&g8K@mfwZ<}7oLu(hcq zWTqsJvdp0UCgsMlFw+QmI3WMzkC@pRl| zCq9xACS`IqL{0`*MgWi$Lp-^bmJEF3{R*8p7xS3p6!lq@% z4XY2gn0w5tc+piv0(6mEg2Xuk)WQO22@xVLfUaOhrC5}-Aa=^xgYG!`%3(@Wyh)%U zBRD!8u4E!{VZ&pK#GpuX^2$sotp=$qdx8@_w}YTwU{(bkt1*jMOj1RMJ4&54J%14IRR^h z7S@VhDOScjIgXpAgyY5t`+KMKB)d4HTLKhT0ZxIEA|aatRf`%qXYzV%y3m@4&(?BL zSE9u|D3(-|s5HnpcI`Zn6mA1Ko~OtDqmH8EJkmPqXj{bVuPSDqJr5?KWo=OHDJdkM z2lpUF69Ec_kVvrgl#$R2nKCuVfK0glxGN@{yn|o>l?{D z-I}0G#vAb7X%XQJAk-t~ppKV7yE2e;h5QA#OT~GO01qNBJ!eez3gA*gpO|W`STD1~ z(m=J0fjyUv3QbAyhSG5)PH}fpf`x)sg7M?6g#-!}9|i%q32?HU`hPsrL5_}?f_!kL zFE%Ht!oj1xwF^zDOx7le4G@FAnR17 z#=pidD0@$yaXED1(*$2;ozXeEKsU1rEi+6{eRy#9Y1rck7O`MjWK3SfmpCWEypXb( z-n5w<@}^ERj@f-B$Ql9V&h1nY;=K%n-UY47lgZ|*MJ}5MMc1;cd;K0cXt}$^?mueL zKqn{yuJu3xLo>saA?N8%Rlwu`tHl{5C&xoQvJ5*qZBZX52adu9CZQYS*cCsSA&Tc_ zr~ZCksCL9lD){7jcebX@OdEvaY7A-CLxf&Q`S!*~j*VRFL~b^S`794bv*I&r0V zzV`E-lTMOl%i>-8QNc}?7A6u!IRs8A;dCS``pdDw9Xv^ryy1$QG-R7_48n-=yOeol=8L10$siRNAqfn~ zn!t11vh>-yc_sx;5)!O*vq8gBnwGNIc%`}*o)&->ws|o6Z-@E(zf$Xdt6EK#*aJY3 z?G7MDKr!%drW|4Amu`WMaP%AJUSh>la(W!^4Gv}kb$l{U+xO;Y=4rQ6D8o+*D6t?A zqhlo-$8_1cCo+$yXb1~j>Am8T(NA)akO0jS>PGeFz!;kVcEJEYebdezK3_!;Za(FM) zAy4)-xeSebvHrce?`?=kS)(Z|{^n1(vv2k9JQ~8hM|ZX49sb{Na57Ke%K5X--79jO z5P$@N9C7$xB;QB~VW?Da;CdZ0SvJ7J91f#A!U+mp`r_n_w8BJxxC7kf$FaZvZ~x_g z|2WLTQ5)e*!WF?*Pf8=yDG2@5XUqT*W zUL%wq4JKzZJ|vIhF_Ev5oVJ5~52MbjTw`26W zjD!5|15BV5u#)Zca8kIGeA!8sD4tXw!7NMwo36juW_TD6ScWTD)C>Q6=4*jc$XqQ! zPA@SYc(Py_?L)j9eoeHk#p1Q{+a^tull+TZ@0CoW{bhF^k2|q*_x9L-06CJ}plSV5 zwj_HSUHVrs(<~BMIC3&_R%Szvo4t@@FQVk*(KK>kl^Ho~16JfS3#O@ZCL{?EBV|?A zOwM_yU)7ovRbE7rRoGdq0f?Wu{O07jGGK5&iyX*a6lGg7V8{b6)nej~JvBW+80`MuT z#s6UOU8^?&0F4B?92;0M7a2Ap*k7mxfW4;9IhnEs$;jJmA`qwHqQ8PpY)Wq`ei;y! zg3Gv9C>Jmm&`s#iqq(CZ*GfSmz_37{nw5od7+V7gk0ackcKI6Dt82Qn;26j&qI6AO zGB2uU9uz;kWC}zNcR=f0l~ab#x=LyRbyg-9w^eB5JjB~)T2`~GsYwlhIL!kOvyuC6 z7%AmTgWv&SAdFLfZ6%)OX>L6tH7FzW24SULERZ~me*|mMfNkqbJ%(5*-0vUeF(kro zTNa+3RJjaj9%i>T=chgg#FgkR9bBrV%^OJLP~F3;cC~1QjR;Hy2_Yb73Z|#=b)+@} zHv)Bs+CHZiOM#a23J}tQ*&|+twgycWQL$R8qw<6;6X9!dwgHo8LrvgY=liHpi#i3t zLS~o0ZAu6bU0-D5^u)m}EDtSrTCmE<3s5Q|nzN&D`Op`-|mi4Mdud1rSrvraxFoF5 z2IS2mY|?`huEUDvy?grzNnCddS4CY#ickwM1A47YM6iTM)(G9T^rQh=%50D&RmVfN zG*HKM?A@9^4gD;s%pC%mEbvR<9$2L4&U5yQOVE}Kr*hGS+GJHk{AWRFv~u1tS&9U^ z-Y<~V2e<&OE{~uPuR!)G1~M*nl2W$GjKwTH6tt{mBjaTJ8u4S=*%{|O&7*@2H8od@ zkC(%H9me$HV)4*B;F7#<%5Xm-TV==y`+ zcwzD!F-J_tQ=BSXpNjpuIvbwos@TF$HNHN zOa)3#5aos^*2v z@QQ*ZgseK8<^^whD>@QjdN@6?N(HK6)3N%RNg?TYp7T1fX=ZC6dzRc7Q-H1bQ(E!j zGAh#tbqMhM6pSJ=v5k8s2qLns=<%YXxD#@C18^C1DH<-;EFgfqsw15qCic0&9rVoB^ndF;?5j+>|P}%1N&@6)J^h5E%jhl;#OIYF+fLVB~Ec#YYRvUOpctp z96orWHHdDcoC*N3hwh&l5c>=)Ft1b%%#x3GfVdyYvzk@pd=VKPBwfecnb7lH2$ST~ zhKt@d$m!lfqisRCYz$#5G~)C}V20deQBg zl%ch#OQ56Zj%^yo;a?zl-V$Cqq#y=`TAHP_#XX>xQhy5Z9$TCqFN!pfAVoeT_dfLZ z?r{p#gfdEY*s1AY6Lr?KESEWHxF$Csi<+5hf;Ny;&d;u4@SKPHrCD3bnGqw@L!MJ_ z*4#$gwSmoF!CLaSmD>K(cdwJzECK`6Hg}q~*LOIc;aT!09%5`ycY; z)wtP>)A{k!zd$O+nxxLe)Jm_f_-dH9Pu4PP7t482wH2fDV>9z%A=)A_hJ&dFpB`av zm@`nuG)~y_hsji(XR$dMtVBSPVS-SUy?4_Fm2#8qxV3lT-pz3y_wq}-r{r$hj|Zcc zvUy7W==QefvMYlbQ2rT?PGSgk&i!~mLHdOkAhIP=P`sgVZt`%+X^<~_r@v>JM()Ph z@Q9z1ujWs`^*aFkBoaA&?FCO9%w*UBt)Y@WZ!X*9J8{G2I}0em|9=Ik*F#9=ATQ^% z>yPj&-~W48d6E16_E6|CcR8Q$A(Ets1l(-$GTB{z1rf?}X#+#~ zS=P&Z7?PKVhh0AcSt0M_;Pj8$+7-Vgpy>ft@tiEMe*Vr#Pstp5DS^G*ntZAL@&W2# z;!*ux9wnfkkx0a7;3W4K0~oMxAb3)XqFDX`fEinJ|I4S}#8>o6SgRj5KCC6F@yo@M zm)Yy-o26fKPXd7Z@>#A4NdQ=I2pGpWvV~009Yzmw)GP6bHPyE$JBR%DbV$CL3}KO* zx7O$#8dz^FoB^~X@}a;7@zKAtIUk;0qyFx6W z6Bz?wZ*DBOT87)Y;#_NhSP@>vZuqG537&CS6!pM!R!s%r{9(3`H@9}@m@ zN6XplcxoU)FQqB%iLC}{!MI~3CTV~^i6|qTlu#$D^9ZI|Cf_sVHK{pGrryFcgQRWu|}m$|px$l2>&EpEEnTA!L9l zF1a|?k)~y)O<97DHKj(rff|54o&k^zr0$sEuS=a!P@A$g@6FBqq{9-}^1BQ3OdvO@ z3f5Sah)7>>3Yu`%C|FQZV!$$=OJz>JURDFA(=)l_)0Z!~()4Ktl&?W3~Z4G2G zb2G5@2CRV*T3{7-b>Gw`t24$u%#$PM>>!`P*%_r_K>;>G0^_8XQb_djN^%mhXm6-X zCI$#9)GcC4=1E!MOKQNmXj8Le@}0|wDpl`>(P9%J$#C?;vKywHZj=yG#e^iNV|2`_ zl+hMV8ew4{Y!wO@qr51l z7mwFluA>bS3H)kxDI?#(y@35fiBcA(`o|}OOQWJWi|RLF(YY?<{I8e~O!pjXlgqwD zx3Q{X*63LM406RAFk{Kd(*Z*gswJ?JtZVHbM~ao%t2}JaP)gHx4hvCL-8Rs2zMiGl zT~-qmG3j-#AtGp1Wikf4rkqJ0VSDnRF&xzlv0PC;hzV*$UqCECC5b#5pG#1iG(uP= zp0OP*8_3jC86m7?ROq*6Hp6?jO+o#IJDs;C;!p#b{*)12W!C|2<`u94)#N$oujn~0 z4%|_JIiGQah>p}k&V-osg6}}CkpE(}>L4|aCa*Fhz`+_g0X+)tFkKi~fH;h>zugTg z$rbr`7DPF1`!x6C-@8OhrS4O(HympqX z@Pggk6(3BVb}cJgt&6lW?Ldge7*-vX8B@}f(H2QGqjpq$4Qt-w0mz);j#ZPcGZsaM zur(sJu+?Dz&1l9Mm)IJ#;>`1uyt&b$ds{UXbATRPflb`dMz$C}Q_fECsAwZnvggUG z|5SCBazY-;bLaW8U37#hAd1K{KaHx+DC4L@P*-3;lOn4kWnQ`vo>f#b;|K_WJ;Z4* z0c1-cnn0__P@*%L%XsbsMWikTuj=)N8KzCu&}ikczQxw>n*~m?>yNV_i@^WILck^! ztmCPyBJy^8Lf08(CA+!S!SiLSqJXE*tF3iNmmH*l9px`pWiv5BfX9e5N82djOb50wdP-GxD>7Mc5yspQ%TPd>g8O4v zXs;qIL32V30V^bw_>#;HB1fi!pmcR)mYK=piD5dR5hL;RNkga{06g+Byl6!(v3wH| zSngW;I`X#3N;=F;LQBW1w#jMw*{XnAKz)VVZ5Gfa0NXhuT?06D-ry<`nW^DdH{PU7 zt`)#Snv{k1OuEz#!wqAQSKjDoBgZivv?{Vy1EN8-_63wDXJ{GewwCbuUdO?_#z{Au z^0y{{dNSJFVa!lxZy5UukHd}^;>yw|@*qt)LpvsVe7x+{9nYb7#{>`_8-Z4@md}r{ z0-8hjhTFMq&dVz}<DKyD;aWS+!gYmIcbB3sU*r`j)3X%!TpS3U8QcdrN0KCk($v~r>)oh0`)@~ z$#{8@%}hdH`83zx^q=H!_RhLv79Key-7wTX`NMiD7&!rpmi@aW7c3@nQjs_Nemu)*<_KMY zAYsK>t-Y%abgEgvaHrf>$j>*C8}aCo8nc5o$N(CZ)`;`DcSan7N!G&g(Xq=^X+&<( z6s*+%SsrqK{8y3VphlwS`U3s@-r;5Bje+JXqR#BtX{yZ7u~2yR(Ho>7izat->B~zI zhi5FMp4U1o0Zl_5M*pE7cfDgggHfI_fomYYgj6>R8h?~Y?BcVul+&-~V`rnGCUl@s zJR5}9&MSLvRKON>I!*?FuVbt zmUDUTmJn~E6M*Kd1A&3xHEYFV3=ID5;H-Z*{EP(BZ8Ci{2!knLb~x=PgUjp>O3#2} z`ZfWQ;mLuF=HWppKdb|6aVLMlSl@MW*F~)PXvd+zg-?4`xK3&ENC_xv;pitV`3|qz8A6;hvCz$MFHo zD)t|N9g&0GvXb079v&PB*Z2*~Yw{xb{s5Yl_STCWHOrlvYdy!c;(vV$c$q;rIlPGj zhL}e_4W7GZGo9Cu^YKxBBsmT92hjNRCri@f5Md6N}nJ4*I|5<)CTr~i% z{AD^lEQk<*$HI#rro;2|+~1A*b@!m>T3s28O6FngpKYP^BQBV`nFY+Hx3xyecz&2b z5GX+uw&KtEQMP4yP2cqohkl&pw_fggd5~`|TX}T(zyBE>w%hme&$~~*9VYvu#^gsa zFdx}$W~p*gu5#gt9nj{y4NLkBNp08sY9a{uU5HJn9|of-z)AcHKrenf!%h%Qzi@#sk#NL$!8&@=nh8!<`b@49Pd2{_Eb|l)RI- z)8TyV=L5cEve42%IBSPZ!T!}L`DVEMn!N2Frn3P*EeI?lLzQ?LDrj0+1+@{NQ#0jG z3h(mLp)!w$;JXPff-h5_gsw5hOMs7pD4PAe~g?ZKQi% z3ws!@(g;>e7V|U9KU23YxrAn4vf~-*JM7xt+o#J6_hY0eFsuAhiQEenSS^V-XkjW2 z?{023`ff7h6$m4Ab7MB$XBK26jwDNc20nx}Yk(+-c-J~C4rxcZw@rVra65E@rLo&^ zxfrwt7dcHdb84$WiD2b=rc4gzQShx1L&uEHtep z`D4e4QiI`#`y^a_udNG+I$Q@y-ZQgC6@*<0OcCHu%3NM!WEB1hm>d}x@75IdBILZxbB*in?Ki}p1P|dy|ND~)&*{0E~17R@VYIbn@%V+4Hnb} z@*?~oKDAXhGFNF!*}I!)AXrZM5%2R_BZ$bb*!-?{ay{VEj>KLr{acriZol4eTyTc0 zE-Sp4quPvD9g0(gfY35Ug=xrbT%lW`b~VFAOi*Hnn=T?MeKdZt8UPN$QUi;ts%8jR z8HDOt{sC=CsWSCJC4nluN;;ifkZ;5gD5-+%_Q8#(85e{W{Ei$g%nP={@1PRR+rkQ) zlec-=Mb;KPC2JuLLdV<`3$O<4UlsQy;vKmuQ+D#7jOrr2&Dxc1R>)CFo{;Z%mLmXN zmej9&v6Z#5Mp=X#z7E=`ENj1PI%aj6ruY5^WohR%Vdhumh!Xj@ib&wqSv9_@+GQCt z`-SE5vTP!ySNFzFi%m&cm!P0n%Itxv{FMbFCLM?JHho$R;jIZ^7j*h=mf?ysrQ#hQ zbbiUWEC4QvJsimbPgKDB|hR;@>Iw{a`ntc z-~lWGK3l=`*8w00XxNmYPv_Y)+Z0_^)w~O?$Qz0lv~$kK%wwh8S^BsloWO-fS*P&{q&O)bw|N+h>M4M+KOH!j<_+ZDgPp z5*1X0ga|MMaA+%DkRg@}`=Ge@)Ozr!%_v!gV~tKmStB!EyJvi;ARR3xUW#bDBxi$Xn)uMIic@015*=+MuAomJgLa`e^gyxP!q{dP*}1@ z-?Uf2!9*g#CUG0K?jjHhqz^Ywo?Qg2W>ouL5T$ZrgR6t~+u?lPxrLX?`BJbWzVGML zFz#H*xCYph8yG&is6DOgvgP#E3AIf%fE(P<9sxheac~r%KjyExs$Ew#@C3|r3MN$C zt}EA91rrJ0In02ZA1xmsbFVyuX<6r0%S3@@+tl)QSpj`alL&cFFW&+&HHxl>$%7+G zIsK~`qSKA%^2WKn$$z`_qLKimz7}#GN zd?GUrRWw+0;D!uh*)-3R{ig@Vhw0i+A& zHZ57jc-YEz3+_ZnXxM*ooCS9mQ=*1bCr_}QB$&HJ(ex<(zb(Aur|!o4d%3pUuRiKy zzZoq_g0Bq95^*)D8EHT9HWKU6mbwFtF>lr`a%ijzi2UPTeYVRFN>h^UQ zN{jxSfYRhqUgL}8m6vQVuW*qx`NOZr>16I5x0kiDhpm_<932SCr?3b=hWYJ#dB)2e z{<^KVZcO#{0kSiCcTdTaxb<=evaNcqg9UjYFY|(*JQ-OIJg>i1cdZx657Woxg~_|$ zjqhDa^Hz)A_Tljv#2tPQ{gipSJjiRX>KS+alU9nq?N5*L+iP}_HG{Z$EG0;R8D5(}LC0xsxI2>PJq>H6p0FnW&8REqH56obD+w6OgkwG; z{z)k?U~F(6vW(6HO0kfkaX%;PaeO$K`l~}1{0Dgl$s74<^ZkGyH-t1l_Lp7qT@S1~ z)R6x32f0UE0)8X2%1E_*H-13LQOoPE4P`M_%$t5YPvfsDGm_}nOa6Ws-rqv{#T4uR z)E_RmFZ}7BF3UG~4b4J%BRj|ey)ocUB@nU?DnBg`DRa(XTAT|B7{kel;TR^7>6jSu z6wB-8%JH9@6mac(quVqO(+51Tf(cHR5(j_#7<@#>JLCmarjsG2Ihj31KFi-FZ+`b@ zLjV+hFbjJLZ&6;MBQM(b?@e_7Kb{vIqDv8?LQ2LEd42YI5V9&^*B~L#g)5>zBh#y|Z_#;%Z zBTvCmLjLPjg+FVd=fF(|WznPDvO19wfu$xd;;X&c(Tdr-Vct&Lhg=&FHhb80{{q!!_oP9FvwzwIW10r zqN0qz3qHygWxOeLmu4KfT`7ySgpc&%qY`G+v3i_qjO@g%Pno&FCuF(|(zTq9Q;)Dq zp*c)dhwF+$E0@Vx_hZ4|A{kOfZW9NAHE8O#=_vgO_d~}E@s(++x}sj&WUSMm6(q2d z<{YaH;^)}$3K9)mTJ}k}wvu3i(jSBOldMrGQ3>U8R++69+31FTDd;Rfi93c=a2Lqx zNWZdbt~S76q)g$uZ0c%O)+y_YTqQ1=Ha8It#c6K5iMDULc6eFxX`3^@xjl>31RM>N zhH-iIrZ)Yq3_o|nWmoXN>uDL02DFQd>=3=2p<3OrG&MX{@s(q8^ z5^+bEBsG9+pb5-CV~^JW;D!lTe2->>osB;t0Nf5PP|Ct3@li_n)E%H=%sK6tGGuMA z^oMss)Fzj;NkwE&d0y958zX_(K05i52GUb8kHAx6RB6NUtJhbq(ZWtI3ulz$%K$=& zW}|G9iZ{qRXiX4+$%a(c1Y#0Yt9maEtqueJLc5a*VV5nUhR`s_^0G^w6`BC0UP6H^ zNhR=Ylr$bNNOTZ8;S{P3D%1o}7Cuh)uBP9hZ(f!m>mX1JD@?7GkTfrK0@HjB*A zIrkHpOxeg5v_QJ56yQKgL3#8<3B*>#`oIf+bO~jliXc5VH%{Y0=4@56FhVG?2O~7~ zG7(Hjf&n(YsB-fa-#?&m++k%VpsRz|vh7FQZ%=LmHsG3jQ+75Ov?jn)u$akGdnpSL zbCWvb#oyVASx7zum(*Y(ci1(&A=uDSO(fo;y9{bSMT@F1RnQ+KV%PE9s1sgq zjR80U6@`LZr)@UIS~Q(((O;7E7FQw9Sp)}tIgZz&?z)Oxl2w_#s>MGdoh_HsmlGyS z`IuNR9I}!s1}v|D7i9^~MEW8-PicXCE(_SqdV>bILE{0S1@a>)2V8(i>v)kxWGWn4 zoos7zK+v4CJmqSEErUwZ0L$8OIqWOy06!`{Im!rCOWFGsO3Cm~@cH5LMQS7Ox+ecB zAQ+MJSUx!O$X^`rBg;V&7BSFYFaj1|me4^%Y0PanyQl^hk*5EEo+57}3Kc#P>WpOC zQrUE+paXPiM@NBk?L9aHjBEsh3o#yIQ+BMou@^68zeu*oiNM^D0XhsOHJup0_}`Ef z5Gf~nDuxclNc#NaLQx>0RF}~lgIxCD1YbaEL~!Y0GyCuYilapY0%CGsYJt`EC3-V) z)?yTw9%a&kwvZN^CyZ#oI6>caDP-mn5$8EQ#&+)S-4#;V-pQa>O)VKlOLK(yX>^1e z62h^%UPua3$2#h5rdSQ+HRa^1g(xgBfi_B%ea921)Y#scKW82tJ%&v02$5SL&r>xq zeD-GW7m(Xz?GUZ}SQ9yq^NNC|fFlY`qO$9Jmc%J9L21kJ&M(8$!!S2`x?Mau;1jnM0p%SEIa^xQ@>YcrmvHXV}ZQ8nE!I=<|4VJd1WIhq>n`!_;5?ZOI-s%h@pmI$KJH!!)vY6TH z*W++FjY|-+`ZjKD&za6)Xs#46pP0E_Ys(Miw$wJf$iCs&H-yXbaPZ zIY~T6wLD1CWxUx3*j*!Jl+yHEp#akksbMC4U)Z{tD zY|Be3K}A|r6wV&K5E1BmnOzabp*KjG_~FsjIZ~j4I4$ro0P9o|yRb6n#;nuJrjI#l04b2U!e_>4u@$OP)jSX?k zX;H$(ix#t#(5T3u!$^}D`K+lk#8E*E z<7}jaAgYLmby`p_4K~&A67=|s+hjAGKLWfHlTz_YDnAY489D}uh0+NB_h$p}Kz>y2 z=ZA6o>9^w}a4{R_`_vA=f?qoyRO~<{Ft+`KZxD?*NizN68iM{P9nYP=@@d;2q zaB0Zgo2T9K_5c$4=`y2iHi8podlpCcY;=g+02>;*ISU#+x`MyHu&3nI|T|Ec$S@n;ahV6zYuBlhs(pKe}N{{`C&NUCWpx=g2|tQ zt>bPuj)qvz+?UOHB!=fwuJLZ#4D&hpW*Sct6kM+V7D|ivmtBG@T$EVEf?h3F%Xmk6 zvE(ZYcCz!}&5&%M&SQj^cE5Z4^k2^aMhywXrW`vDB*~IdKHj)>OLB%&C%c8%ofI%= z9`|cVG|gMN#;fTAo_>9!M$^zgRj{oX=+(&s#{SfAL5TPKkI)fM-e6OZIcy%r{*&$# z&tJ_(!c6AK1fDENblL6Bp%xDTR|p2p48NvwF)QyN%l<68!k!zG*SO+%%TWIWdnp~QBQ)7Bsr=akxE9Y-BhkII}=Lk<>@nQjxt$=uv1>;iDFvSa-B&CLg$4w>mNj=+i3 zYmxA=AgYfQo-E56F5@V|1#qx5V^qj8-JYrTQKTV4xgansE*^bqhZ#e&FB8yeS88Tr zh%^GLvw(bzx2(~MimUtcw&WcAN>~X!6j3%r#r>tI(NKUM)MJhA5xJHgS|Z<1eBjSb zL`8+Q&>*7}zV}kpD9@R4!^7*upl*vc!Wtqw?%b;dTDCsV5Cr+849wJ#bd#21yAt8? z^^90ZJ3Fx3Mrb`gIQEx99+Z=60Dm~`r7EJ!E}zuNC!ukhf&u5YIg>JFq9gxB3eXTm zw7jrct3}>ql$nOVr^66hwrF82E`k`LzfopOurkQ z0|%G%>m;0H{EJz)ZXC+Hl;5{*qcK5tB1WtcI9+N731jNY)l=A+lsRSV!b@4RwvGhA z8>dQz=jEE)(&^UL5FMb*&BN^4w$W8taKEB)h^>L_7EQ?&{J|xqWh<6LkCG+}BB-r$ z&TPuuKDyB~){yB^4Q-XldGSSjfZ5KtvwY`hgP>|3NyF*w7f@iMtvG-JwM7ZFfq33Y zZ?eb%4vpxSNuXyf@+#$o(|IUE9fJISfs|#hY(6vaiLAtMsx}xJZ{Hp`37|NaF*wO zbnHq<3)3FCVEh`)Q_8A^%nYJkXMh_Jm2wP=z4 zq1;}Og9lq%1kk>!i8Xl!Q9wHPLY20x5I2OS684Vn38Z9%4f5#Kv^FiQv~lDLcVEdhr;1GR62_F`{CUp0aHB{PC~9jLl)>YOvBOP*2;a>kb4O?Fz= zm30NLZy`_@D}e)GxN`^;sZkf^R$pJOs&n*$(t9jbsts}%?9C4Iy-VcP$XiFsX%{!z z)=UPz!XpBF9>H`!*ho%Iayh?wTAmjK9^`Oy))5HxQmPJV%?J~K`WRIwYllOi(IR&$ zkVEj|3Jd+>M9EpePD+>Ra3)R;PpS8dgjx%=0zMA&QpaW0FwA#_1g`3=V}vZ2p3hE1 z4MI$e=647vIgT@C2netxmBgwg6GN=DtRS%z;R=^?*NqpEigs_pel0zd;u0a*1UtZB=HsABw@$V(W5vTGO){LAoh*f}61>V;WE%gycV zB`R^e((lI878&YAXsu$F$a_PK?|w+OcLo-Fv{ebTV}uILf$xlHG@lPEcr@D6xURT( zj*i)?Dr?!_$(_ACI6<0H<{8%pZOY3Ul;{9pe>cD@mIdH^L}Fs1ZM{e#ehYQN`zDGK zWpCF0aA&*Ie?wu9SBzH>DRWcrW#+&@0S-#XOfI-C=$BcWMN~Zl*e>d=kzV)nM6Q}*~)7;|HjbBH~878~*G zW${rd&~%+z1E4*koIRFfU+z@=cW!K2u{zcG3EK4cu69>Ioq^uQk_5Fz&>N3r_f(z= zR2V7aD?9~iQON|N+}d%R4ub|Z;L z(4t7wHjM;ICyz(1o05CPJm$f$1T*hZ2z(cJuFJUai|T~*wpm%rvqPk!4UHzvFE$&mCMnzjqKeuFuvVk(9#QFaI8*4P zFcp-CCPEU10Syak2SUjb*hEUKm5$>%7%Sxw<2m?Dm<7fjIUfH8%r`f3o!&c)WmP~a zBS!t|cHta5Jp)NY7-mtIynyzmf}#c)_O=M@+pdsYUPfC*6y$V%%5DXUAjVw#qMMol zM1MVt|MAlAyx>rW6F7Dxq4H!+TG_K0L5Ah3(44j|GMe z0TWc+B6Q%)gB#5|ro(%1idpz$$_rT$5GR^d7Y)mmIx*0!t|VlSDCs$NS1_El1q-^* zo?WU%xr5TwEAcV3Ni~${b;UomDcigy8xLsBWcJqFFW_rs&ZudaV_hN~cOwM=np~Ao zDR4QXWI|BzV&fZYA$L)6k{3)ZF6$2?FblHM?#UwoiyB)Bc{^6C%*l{t5@vAg&*Yi3 zWUc2hM+V{r`qJds&Li1#o00M~RyY+&A}nOFl4U|KHFPqK_D#!84*PKd$`uHJDZ&n# z7z`w}qD~6Wpeo~e*+#hCowTJGfW_?k>ptvUmNNrGRiwRo0kCCh0fw^3j^LTFP$-Ca z!&{fC1;HWkK^*BTIt<=cO6SqZKn6XyO9F_tU^e6haSw}<6ZwxQzFGtE9A&E08M?ya zwP?|yQlNoNr~w>ZhT<*k&meVM5Amg{&S&H_b#E6hE#II$0N_pTx_7DMzZ({7cu zr9E6g0gyAL*P44A>|o*q(#paMR|AMlGgGP@al>YI=L;xEo0JQ1XU&YI5?6upmzpki zCj~MVat<~H*Uxr4BocdCN-8sWTz5;D=a=57+19Y(ECFfOO@YhIF zSxg1s2%xwUANVvRr^`mpcT1<`$chdQSaTRYo{s&|a0Q72+5eLf95O6km(6Ze=A8gR z^Txor?f`fil5fX{exBrij+;-v&8l`K|Ng^t5$`7X^9JBdIcqMn`RCU?AQlOVnEj{! z`|#;E`PV*q4I&{kZh=F?A^B=NSR$h5f4&TduWyZN&8L6fK29eGs(ag?w~xtJKMb<1 z@1CBYF7tB(8x!MqpMEe@cG-^xN*!Z!)uh;2ADPipD8M?SA*^pa1bP9*5*B zDBw&FhvE4vxfl|Nm-+m)mCO0vpYQL-?Pwr5r{t&Ous!$N9rTI3-qQKwWFcHfaJ-BU zw_oT;9jCjzO-~MQjLZjik+D^OOB=61q zO~2X9!v{kUhWRrB=ZEP6HL#(-6ZdsC?|dO`v)3_-P*z8*Eh_gele|=NI@y>M)@1Gl zoG1tR1eO)bJk!yte>=q|mB@NPR?Dc9+eeqHZTizn4(#Q@l1wcR`Lw*bUr{e-Af>)(Zru9wx|HJoQzvZJgkJkk2Mlkdas`vJdWA4cyaxCamZ z_0!PFjXCNxGx{hC@?-(R;j(eMV7&#`1<&mH&5)D^VIMTohFK@hALV$u>!0kr@{|LA z8xAc;>k^X`1WHp)UUn>xZTR8LcrAVi*A`6WK?aMi;^;ack^-?YSAhHbXO;^rYgkDd zkRg?*lIV$Ps<7nja4H8b9mn^h<(JBlB&1k$kW{gJ6^zmA4Se99R~*hSE$Yo zR@HSGYeR=?(wemrb#cB>-6u84Bc0f8-?t`JQF2S;(U;}BtU>0JXZO|-X2XJ78rKvf zU7E_`j3|I@9n7UFY9_moDY$bKT_wKU=TqfvSv8#a->7<3A#b6e!q04dF>8X}Vd7bb zzhs9&%tN#9QFk~s$+D6une5hREU*i3U?NfQ))9EM%zN#tTMd}nBJa?_)EZ z9|JRg%gx;S7}lf#A0Aad9~`Y$fd<1!9IpUHK)Sy$mb!${RWuXeguo=cX013kZPi4k zWDKt(hqg5+@`_vgH?Ehw$!m#TsHz`?64opQ7ZMd5!`i*f>n1BHcw6z_IO!}TYPh*h zGY`=A;$U;BFVv{-vyA)X1|_P!zR{p7xUSl?WPLKY5O%zTMtp?W#IZDrNdY~`>)&t> zhjeX0@Sn-CQWY8U&;L3*P;JX|*ITPcyUBRQXZXGKZLLKr+W?odfP{3qS0Vtz zv}(JmF6p!l*)5>(RWaD<;i98W9jWc#+}QjC&?pQxRhQ3MGNT zDy`%?C5u!joTF}=ln)sXK$4fxv0UEG%`bMes&R(V2S*g@UStJv3MPiMgC%GunH|u$ zhicIp`VD@K?h2SHRJlGl*|4Ha880h;jqp`XGB6-8Hd%2bxqJiyXRl zW=9^rXkzS4cZ((&>scoroehiJuBpDN5;9=~v+xN5wJHie-mT;43{s|RahzuIO`DE2jXu~> zVDZ?KTNg;sRWBz2pEAAEC~LEpJFrX$xZ^j|~AXR<3YTkVRLD`$^SV^U@sxLVSUaKCn%?ScA5Mhe#d4 zVQ+FFn_bE{sZ3e7Kx$3Z|5a>n?My2*v;9s^`LZYs!w2mvq39V}lS@kdua(>Ktg0ER zH(hiTpdc1&)m)piX)<2o3-BB_%BRoX`@n-m3G=izV|9nBLowju@kOo!!lbNx#*Le`|I5VDb)o(`5M#E|D_yiIEXzH^=( z=jo%THj>vwC)H--{#;HmUIHQf8~kC=^X6?(DT%}*DyZDFDiea{x@GQ~05Y7H=pEyA zv`SH_B~IQeJCi+hizv7X^(z-EOS17FY)g zcr?p1IIXa@=5Xe&A00JJ+sfOBH3@mg$@*Q2A5a>j5>^J-uI&;4#Ei- zIWs~yNdW?04qXD2XI4P_k4`5GMwHFo&?sHcAIL)uPFF8&AP-3~cK~67i5!y@+#_?X z2B3=Lo+7zPMdhbZsq6@`upQ5%XR4_=;c25A9k0rY5&1#i60mp?8R$n@kdEl3Ey1hH zjWq%6QJz(Tdy2C*!|q|!olwFcv%S+qK6s>nM9oYi?0XzQtHbSbD>(}oRdNKe4#C5) zba@*gv9gQM`%bbfMhAYVWim79AQUu6LDAfBM%ixyYE_f~^Ha`WB7MW#pjIhF&*^YJ zOz332bD#o=##lQ!Y;%JsU0LQlDK6)e)1Zg9T1A=|Eig<^dvHRfe44Kmv~u*mFk77* zUXx>m%CVYK+VX=^r?r#ACoMbfTi4N6oK13U1IneW&a0do_ZOH_3ZXr?i`VZ(F%lrC zl;ucp>Bq^PKZCa=E3KH&6lamSP44Aj zwe&Od*04m(qf9qLmXrKwE(oTLK6yPIw%|gvN=^ph0PI6@zk53N;J*2wWH0t7l7*;7 z%fLVOXYq#3^WF`rpPW z@00$LYeb=fuN@Y_m%VrHLLtavuMA9k1xNIUPygox)XaFcvV(Vty#ZN& z?+jhzI9!&0f+p953;p>BT+cFKh)%?*K;>EgF~BOUAK}|+J`4-t&s+I`AlU(MV%(n1 zx);x5v{ipkMIRnAKY?xuNvihdBivF2`#9`;5`;$KX*=nC20_! zY*w8>@p|lBGiayb`FF!X&X`aCBG2#NxwfVm@QGvN*#NwCwtyl-Xz1SZe!P30wP_mO zMsI}^H`DZ9$%qb@({>(@XZad7i}qK^iS-s1P=HvD($Jvj-r3rA2S1GY$``${CXeJf z9?UBkv;bb$c|;c5_P}T8t3)Z5f5EmKia`-W3kQ`sn1~}o*RVx78I>cNQ(&R$5gCm(}+aJNZYo4QF^ z2K(46T*D|O(2T(GX2(l$3z?!_OL+#a0V3M#oY$-n34|IoIi>*6U-a&PkY;6cJ=v1cQ|K3yh6giXeTR-KQs>nvcu@Hq_pmm;R9t(S<_owmESvgk2U9*gJgJ@gO!=Zk#UYWgNoOx$0Zr`N}I;` zg9Zf~bZH7H7tVXyc(3j1ocUz&mK<9Rx>_Ub8#s!sMcYaUMa6>vlwtd+sCmn^^|hGq z5$b!-!Xm%V#oSpbuRx!YRR!IE??q~5F2*>+#Kkw`4G7{XQk55IKXfCVFlHNW1CdY~DL82wWA;t%nUNHAJ+S z#vXx@9WR@XnH)S4sYX48lk-omv|P(_icq~Q?x(S;4BtL^Ugl+atVUKv@KVb5T*pn{y9ox(htw&i}wtv;cQ7A*4<=C~@P?YanlV4mC$lpV9~ zpyl(?jhC#dbT)CQ8h{R-=h_Dyt{c!H5cNsZWjdZW9Rp%bh;0ojbnQi09rXUoh~VZb zp^U~@RwD4xmfE15C$hJOWE*l)G|cDcLKtf!6FQA-vX2Vp5QnH{Wt>GxP0re!IrDbP z2~ef-xtVn_lya+TRtMY~q3V6_Y|57ABElg@l9A<{M_Yr;>5RWXVsaw8DCYPULvJ^jF0Od%NI+TAy8)~FhjO?Wvqv1jg*dur0;Df3pYusu2Ot*nX@c&LsFqAn_y zBo6rH6!ei8C9#y|ppF~yoEnRx9N)@oU!RT6AL|KH9RzsmU^VL`Yqfth2KS^|X+f<}8C}mjWcDWgPEg zzjG`bXiDz5u)6bbY&awpufM@*JT-<{h238s#X$!5;pP|M{&`B zsfHc`SQa}DZWczJ%lUP3N~ z!?NkFQ50yh=itwM@}kHKPK~V@A)ubfIy)ids)0_|lmaz{P&GgapMgsbOOhMbS_*U* zq5I5GOS78Lie@Xzfq^Ex!SHLZySo@PWz8Hv=jk##awzDbWf93!iPgNAlq^<8->g*8 zQjkFNzOdA8jG5f7K}dXVKXYUkY2CFEA)|mjOs0HOviRAGsx(ys;_H>6J&y*)k^$C7 zIY5tcF@X*H`T_hzyu*x`rpfbsFtSgqxv}!YX`_#+j3&d(wXA&Eb(6dBR7C?` zlgN@Efr?sw3_BDzG(EuSTNIQa5?eFRV!(=&ijOOgMp=0|-!@*U2^^i=ChG}MWNS#d zGr?q@)gVP@N+jxVB2Hb_waluu9JFc%dESWBW(Z?OmxIjl&o&Z(%0m2UCiI;Kcg~6l z*!C&#cByoSW64rv2q;cjLz}?BRj@3$mVy;9lY-(jjmX>06BzjYFQy3PIlH~E=82~X z@mBJ0&D24u=j6;}Un*F4?cBfj5S*?|BRrr>A4L74K?d}1R-5|oqI$e8n!oM;sh<@N z5tYh^|Und`XtIM?;&hAFKGBF4j`;Gw&`DOwP z9}n;&ensNb^{vWDKPQI2DX}5)f}WTMgT*-+b+=v8Z~MDpzruG|T7>PR>|m#W(85fB zX$GOW|L*S1J87|t-S)=wQ~3Jz0^c8!{_f7mdO942zn<%hG6*ESMPMS$%MAnNri%0`O&)P3Mh7 z3dY3UoT1bC4k{y9crRYR@V7Nz9ySJ{CVy%#`qR0;bE~FTLZNs47HBisHFaiLbvP?x z<#|?qx!|8Ia?@k{pvrcHq#UMdo8G2r`urq^wf;_kowK`<>Um*>waK^NfBQ1|do;f! zuP;Xf8*)@3=En~5E*`!5m4_Bi`~Ag&wd|9BGaSrZHyk#XlPY4lH+8#u3FgJjfNvT8 zjB$|PmYr#56oCk z<%>}^p70k8R>}W@y;EFX=7k}2(%vr*#T$Q3eo$5ww+c^><559T=E829S){qHFLE^n zmS6pAutGOC*u9?_G*DImQ_QJrMFX{6$K2m*Fp&6QDc2GL=mMP3oMN;9Dg*RLM{Rl` zJV5;Nh#2}w_d{b+N^DMs^!0OlXI(FkO1r& zbf`pB)Ldhb442HAveRQ;O{%V=xO&OoOVras^qCV*J1067VN12D=uX$lum)3!!1u6s zd6CwfMP71{9bP46&YWjvOk`a&-1~UvJXCAbm>xN<-f9`I1veSEX(Mxnc?&^EIqN8zba)l?P0xy5P09#=ta9$35JPUE@-UjexnhoDYXgPS5}G0cGDq?SFh>8S4J z{6@6_kd-_g_{?^^?jnWl6_}vLHppV=xP;zn1#tq(V5;U5Rj8ymjlzUjz12QIJ88vy}Y%&J_Y@DjSyv&l>$2?F*oKHy4#+JK>+O&o~18-*`=LSRW zOyodf20~M)JRJndHCbeWOpr^tN zA`X~QW)jq~uBB(6D$~5}QpR2`^}(X+z?Bohmn&#U4nbzsR)WYh8=a32UIP#sNEPpXcsX7u3Fvd; z6bOiBvb5y-y|`lvfBD+&xLo;0GZbg`2>udAF@ zctf?8971c{jk2UXtyKjYMLI83TiuL{x5s(AfD%Nain3tE%0)65?3gY0VjxCx<2eJ~ zRs;ii(SlTovqcA|CDgPPd(9xd@o?uHTS^{jL~zd`cNq~b_8liOKO};pDdc<_H9}S= zqJG>tfS?=_8LJns&RDil1|=eLdG zq(el~5+7^=C3EpqSZbmuJS(n~C_*f5Eia-L)>fSDX|QIx4`pEV|R%OLGFtg6cMoWqulR|AKzP~A}1 z$U?vbZ8mOQMYTi6#pR-CcOFzMC~ss(j+}8$Tu}@_1;Z^Z`=gue(5Yod!=x_M0MJId z#Qw6|xrv-Xi=Op!TuuQZB{f>Lt{-x`xC>M!kR^ZYX@B&xNYHR4&XT2=Wvw9d$fA`i zK4dxM$}D9j${iFa)Ix4IODR#Zh>9X<*4E@~uLA_f83byWNWX)p6f(j!5vpxH-Niy) zUqm-3U=#vhBJ--Mew8^i&FlLKs(Srs|kP4V4YAZgO@T>aR4VdfOPnk*3bF}Mcc zxa|4K{*M~GwLxEBP0egTMV(zwT?#Og9NZjiTu$DiK_?Q08?qrC7I>Aop>nKI-42X~ zZg)hjtmHiNW}(YLKNEUFt{;0m+^J}%L~dDLGaGpTBbO*KbMO>edB$Mq0R5n-ZPk6b1woi(nyw&M*NrLxijtHC2S~8s(Z@mFXn*<~Y{g@3ENyt8>2 zlK=DvxvZaf$(x~n0M(9t5Qlv+GD7D>JuR6apwlsLdKeDi)OuQ^Xs&+f@7!Ic<}1|h zBj!%7Mqi(~56B(i3)R7kHt!UwJ`bKL$zYePXD{D+`?TE+iOrt7n=mPd+vI1U<3c8_ z-A9Pn=s6H#-hQ&$Q9w6X2oqSnun^ptam5by2W9S(8~xdAo`!YIsrIa3RqjXhlg-oR zLC&+Y-qi2IZR5|!;c%Ng0v@!cC^NStH?dV-F9ZIXc1HgQtgR>AL*z~A?YuS;UV@1wH@_<~i)}TJhjH+i#oem@@;>;23V>X}YnM%<# z+#tK%DJ>A(SFWNgt}>`lTJc`##Pwj0HY+*fp%+ksjm}^lSaCfl6oP)SZ9uNQio+0loX0D(PIMy6O~#|I}LA}Qhe*0;%8qoXbQA{ zMC1VMDWo=#bYUu$ldGbdesFTV!whOI+c~q=+e6wJNKitb;pXPaJ_m$1xWyNTnGR>- z-Sb=QET42;%Xt#vM|LiTHNqVhIeEZZQK-wRsdzGO#`jLnrEc4j!bo3r&+Cja9;?7v zVzQ1qao|?iVTr0KrWmw3RRM5E(F6`~JB(^TMgT>x*0lcN+x=#p#5!&XHQA z3~7)pQLmqsz{$n`-mz7rUDiRdXG$p{uL9kOFBVs1e5|(A9NCi}t#N z(}K>Crp#8?Zr-$Q%K$XldpaGj+LToc5l!0h8f+tkhBjxYqY`N4_@MkJ0!lk>h#Di< zw4;^@WmU=fL$Caa;20J%6FyEJvzd(@DB_7eI0HEu8Y<4Lda6r^-H`_|4<}zLkj(<5rt~X1T7W!+ zQ?|ZXQNV|wGSpg3K+dq7mEMBdPYTr&!o;7Bz0d9!0J4y;1NuxgF#n;V*($Hup*X}s zW7GaPoW}(?2U7T&>#{fp)S#(A=)(tI|6!V4(Ni3gI?Wi^jmJ;}*~7T4R30hSaoLC) z4k{PbinJ6DCX3*ZSN-V#eq+%6QrjfbsVTD=&7h+e9=<>0;f>B{8ow}TF zS9h!d8fsFZ=@tbY){`1_&WMG0R{RlczDi9hQ|0HF*y$8z5;lvEUgv*vJMI54=^+5}M6lvtQe*V$s1%rFHXOn$`@UE}(yExvG z@4x@!x0-AHemf+8oVM?Wr>orkd3qSkxBfVuA5CWW0o?E=;eL3sKNKW3{VV?V`&Tv# z`l>&@AI=GcY`qY?|F^D7KAPTH+}@IXRRVF_Oo^^!m;3$KUSE)H-I(84Y9eodeBPNx z=iw|nUm$uE-uh*?DhJu_UkT;9O9dToC%5`S4}l$_?~<;touFMla=Se*(} zVVY$Fm6bg{Jbvu8zZc>IXZJ_R<7H!K&8Po$H$1-^o@*)V?X&?{#KLW)F0ky!}$-dzg;fvME}f-QJIABYQFqo zC(mcE`sd?$+~LU%@Mh%vfrUbG^w%cY6J$}ERhZpoO}#vme$2l~;vTw3kxZ zd3AF$+hi{Y7bC3LLEmW(;lM~|oU(=&OXb#XD2f3~f6iv4bI6f;SH@d=dR%$cb z2i&YL#X26;d1vg0Is=2+PoCM^+|lA^bAYm{hNQm?QO@0a{}pF>BF ztAxK1V*az%bW)JbQDoC^wHQ)JU_)W1=ASVv?>>@#KfTx1R=~Tc^RKY!iLPxkro+Fv z@eQMf?w5ui6>-y5O$58OQZemH!{$bLVR!UeqoR=*_Ol&Z;y*_+@ERe5ub)Yu_rVFS zW>v>YJ7cEfr9>B83sS0a2&*w`k{seX3{E8)UWzbHgzpL)*yzTanu5m8{V`fAh^Sq2 zsuI`N5@1lD`chnx*qd87R}C^?y5mbI)CQnk3B6;zDg zv}~#H;M3aBq@c!YU%+ZW7C>jqI6f-JdkaDENK2$F5OBFCd0MSl)cP5TsOQ$huR-p^7skdISxGGXcI zMxnJ~OzSFFfKHE&S?j1`sX({W=6KY7P&WT!;RYt$2BvSg&Jogo zwxo?_$TZ_liw7H^7>i2cjFxluvdTQCd*qD#IPN#Q!6nKnWFP1|&rZv-stpC%^-OiL z?^V?{TqIppHo$CNb1l~Ut!gAY>6Z((Rh7yMqe9nJ;<756jzz9l1@sEa-EE}leAXW- zQ*b$dy6Z*+`XP-?PLnEuX$`b~?l$3GHv=7Ha`EaModV+1nCmVD97%6To63hXCWvoycmL10%p4 zWnQ(m%;kuwxl<*`&z7K5%$a@A#$0NWMI5D@n*~cC7UGPOqobMV5{WQg#=%KZz#(B( zZJBSbb*{*3rmzWOJvGbGfqlIn=x$ZCpabAW0bj;MVXcf5O5HMrd~ZYh`QcH^QReE` zzwJvopq-J2B-WOtDp!KPk2q7DbI%{nItaWo^o(yRh{H3_2ydhU-!h1i7gKxOOR;4U#%rEj zUy^*1ibARYx&;+4+W1%P7;WA(+%4!+L~4{8cs)5D559>?e4l~|IMHyTUl~nXW(IV66h78cxLUKeNCwZ_Au6OtbG z7Mw-zWTx3D()MIWQ*d_2c>SHQR+grCPCgv=o};*fbQ^VU2D%qAT%xO(vK{mEVRT)! zpbu^+-4m<=wZfXF$GN%jKUFjxm(P%AD_cJWKj2Fh$;*%ZEaM+Xfak}9X`^re z{=&4DU={qg|MI`t!b?@WDE^(f?|D2f=z3dI*&79GTZOqziuB*%|6G?DZj&r~k((jK z{Mx!`X)b(7ps4_I;P)tTffCiw#~RgY>;3Da58%0(<;wC0plFpfWk_xp^X)uJFk5@=r$G8I=~fgb@Qr#m{MzxH7>ziE7bafdBpS|KaXUmLx~AEwMen z0us)Wh-h(-#ynK7A|fM)Vh+)nHFOKB7jSd9J2K1+mNtZ^C#6LmxqIu!1(&sBC5IK? zlB)`?`4ROcyyxmw1z-n}uUH}^<*`Y5mElkW6bc>gaPB!;D}kCFaeDlh9E*aarBBHi z*Xz?mbm)JaWP_0DI2^4o=-FrJ_{}N$TCDcV$o)%(;OU#gas20(kD%3c|9biogv|SO zfBeb*^wqFE?jS%d28sR2RRMhoa^>#1iHC3c?XcH*^UUAZQmpvb3Y?Xzps$Pq5_^xs zuQh#jb=Y-sRo_sf8PT)fe)juIO_?3E6iA73HKq3E$K77Gd4^ssK*7Uh6`t-+j?mmESCzTIWxkXs8r3v@-ng7w=q0v~yFHTUN7U!6 zkfQZjOwarKAP2wf2+CRu;NE(YX>|LEEp9o><@s1e`(E0%*FYu|FVYMO*=uxghxxR< zLuUO(UdQFOjn_FT!+MOyFhoATAQ}?e9C^A8`v@53~bioVv%0n>v5EpYZn0K>|Kt zQOMFq8Qu;BzkYeUNha^z{+*RFMU$~ip^u0sFA#QNt(wWn;96x>88Zc{Z{;id6CjUM zjzIe0KiF6+aaJip`8ZgLEfe$OmqetK3=Cuxo^+wVMD;)zTnT9P@uw%I8VnAmbK9o3 z6i$M=PzL|6gNzU=X2c3x(41WL#s(XMV~q^sY3vYf#Hgek$yktrOjS{E#fuLcU7txz|5H7>(XQqI=(VA@vWv=^JM6k*8bRi#bC!4pWfy8)Xp4in$d)8nh zV6pEO)`4rF+IowDuwbZI!)!sk_2R;a51^DCbG^be5Yxm3=VkOsdRd$u5N0vkNx$~` zyGdS9Za|F*cfG8tjCB=9rx5CO<$}0v5a?~vioxL_Ztud)$9$vSn>TU6SlQr#8uvo0=9LC~hEkz?om!Ce%` zrlP69C7u)KXY3SlE%GK|x=EO}?s67` zKy|^fday7xBwINOZjs$8l>y-LCb41cGMG_`r!1t9s|4v${*LeLBLWo+K`wfI3**bX zr>w~?y$4LsT2zoYrfB7TA7+X)Nf%Uo!*{YqWu5Uh?DJGI9Vt53o;>+!T{jthVl$6@ z8(GejNA^)dxeDb``iR4i{w)!wvUrYtO#-V>3^p0=cQ`+esXm=R8)&uDNdT#q@9H?n;pqn0no7dwy9 zH4(chp=(ug60QflyI{9n&(S%7WQ8ytm`pBtTV@W|R;97AS?L9)fSic1>vz%cjSB`{~$P z1LYWtVrb4cj z5>vMSSQGGoE@Zw^yIhD7&3ut?ZCwMf$8hQTB$J}Vrd&e^lgzHi@d6Wzcx;bB*|=!r zv?j#KEMHncBAqp@13$d%vdG~r$&mUxELe8aQqR0dvW)31Lq!&{;uQl5e0;&$WY|;* zwy++qeUYGS_@Jp|H_C<+Y{E2w5&Q99MCJt-1O$dqbf9D%O^tZD25D0ltPR<1-}n1r zi_DboX_||PLPNIk2o2C8rvv~b5Em-#8m2Ixq+?XJdf^b9=BeW<1AEr_O-7Pe8B<2J zO>~_busD!A8DeDCsFdkWk=lKy)~7Gg6swAy6$^4&ivldn3qHVmi>*tGIOme4a4NY* z03ne979!j5XKF4U4>_z1rK&k6IQ_+v!L`^o_CtH)OG!ayM?)S^ToAXRWvzV2cYE-m zx<+|c%3LEG6TaO@nguKb*UMo`_Jf{7IE7wz685xav9g<=ciB^~f-``>okfa#6It(c z0*$GD=vXi*+9=17Df!pJ@$!?o4jF3Z3gV{-HB(5DFVGp^?ECj4iEWvbG0(wJbeSdM zf-(kXB4RF&;EFd3Fl!5J5(#VRItcAZr;3Vb!l^pp4{IPsK36#H$8>*;G&L*X81Y+-tzzws2lhf3Ru>DpXq z)>_2k%F{M-)55p%$QwE8@|rPebKpTyf*ywF>#cpDB3;n9-UsgAys6@XNU=vCpiID0 z-uQf;cJhgajaN#S9VjNslFiPp!kOk%dF0|%UH9|*2bOd>NizoY9tVHa0&k1bPBx4; zp+$ubkh^Q15Hv){SHt)ngTA_~XUdFQ?^F4TCF30?yxAb*Oe#W5TwJ(%Nb$14CwYGN z2al9Y6PP7rg^v5Twn`c?HyVm1J^=J)4a%5xir0g$+AgGr3q--lSbNIZra%EX8SYlZ zTbE*dk+(PlA3^a+Z1eN678B6d8;g~Z=M+zvp|_Bq~S}TN(t-Op8VCK;b7&w23*>(u_NZJ^`V4^LQC)3mw)Fm z!$+>y!z%1}@(OZl(}8-^cL!g;UBohRxcqz5hWc=s8En{ZTQO5R4+)Z4fq)>rhxa=Z zG2CVSamCUE?3>6j5VKvkY27Z8h;y8=>3`P^cl&l{E^?ApOAMHr1+xYS6)@x>ke!0N zH(@;y2P!2!l=G3U!9``XI!NqWv=On*bK*EKhl%MAP=iK zMX~FKU63|MmsqHAgulgNrz~+Y^p+%u!O)sA&xV!W#0A9O=)C%A@HDUxp#cJl7Ed6QL}&QLd~AIw?L+k1T02-^=M{M&v%BB0}}5?_hqgMoBFsr(5BCD!=9YdtM+ zmQ^w&6hb^4WvjCmpu%F#vwq}CNE?|a8QD39`_rgOpB6FnLAdhKyLGW_)-j7@c;C;p ziN&)k*dxCdlj|y6K|NT2R_YFWt4v=o_$0 z*uS(>ue$d1zU>cx{fB@0t$g4QKl`g51JmgEA3A%ye+mF}!~Vu$l#n*V1WJ;0VVZbQ z48{wb4S-iciCA^n-wBm#F1`7(i(bihz3Yx+??;&#Ao@@Mulm03H+>uK;#A_- zvR)joao{mn8SzJu+1?Hh4hW(ex(E5=h;SF)2A4&_Rb(v`%8?8ak;z*Q|)R#(Z=yA8@GYw znOlo>GOFap~;p9t!Y#yo_xDTR^)Wu2#T%S!etj{t70GhbsPEP$h_5k=1zY z4boDA#FfmCX>orEkOY(1sb7?!EHGxFYWrT=B5&e^RhI2bx)yT(W&xq#%F|J;!Kmel zS&~5atX`tUau>mXzmQ#$)%wIqZI_&-n1G`Y!_JnJ9SnV-YTjx6OSm7%al=C;do&=p zQ1rBHjToj|2qBx`lI7%0W*yE+d zsbQvo5b_r%EU74BGmHP$rA#sktNLx;nj|qx!-gURnIz3ZEyX%ymuleK$gAA=z*f~X z1&{EUyTtXfq2$y#o~Wr(=7+n}lg%{%-y}s!reVBFeDW$~I@$mptIBdg#cBfUDN$v3 z-qCl)sw|(nrtH~TUg79LL)Ay-lDmu@jRjZ?U4T3`=QV>vUMg7xD$+eJEu3-Gs=uFCp1AW98H%pflb+dr)Ve-!cnM~S4 zJY9lZtNB%C83X`yijtDrfeRIsC^Ui?Wef&$JIGFV*Lu!Zh5-x6iIufIQJj>yj%p~vT0cioDrc6#=LvBnERKL)UH-_!`RpRZr!hj-O*b`nFb;$f;&Ym zDoTN4ujg2mv~I-fA^hSbOIyp_U{JG#sRA!}Rp&_NE-a{!E(U~%to9j@4E)UrZ^*YE z9R!WYjER7qA8jqf0i?WwglDd|*M6Bp~Bt3N$JEn{#1J_>=4t`m8JxES$HU zus(<{(!>p+dqRr3iqnN-7O`A+9jhs(U16rl3+BEBvJqqueG4(!5-yFJWC%e{M>BK` zx4K3u{Z8rQw%Z3mR5_m(*u8H&hj}i}X-ecWNs{`@3l_odUhG_VUF)-`mtxM_SY=@P zq4Q*(?w3uJ+`1VFgKVP}=Wb?VxZ)5&3v*!)wd*O=aG46)XsRHSvj7kB$c-eG{k33y zj46mW0rsL`HBV>yxF(6RIxQq){M=P4AXYe@B##pDuX5%^6I=70AI}85M-4~DCW#?1 zp*IY0J3MZ#2QP(M0f#}ZTEAcI+gmTG3TAL-q4VW4;HCf{32UQx+(4WaIV<4GfLXq+ zsl+m3?HA507gbWT5_-Q0qM%J4vqN}>l#^pNUXXhT^)8@VsF}hD-L-sMnTmtMY3Ui2 z1~^MOGk@&Xo$e3TNQ_R|9VjYuA98I)32H;y1rP5msV&Q{Di=7wJjNdRT|h1_By?`1%h*@R)tF7j?oQiurCQ&#&^AJVci;bkji$z3_S+B;(f zNyYQXxeRmUm~cYuEZ?^E3mB-PG-tk!S*1{%aV5@(D3hH;LthNm6N4 zP8$8+=XtOcf)-?t%AUp-;4%hUukGRzV=mCB9S*0x+E&xd*s#AA|7aZ{t&8_|;)`!T z{JigX_xQg*oI>j7c37{62iJ#ox7!atDn2FJwJzh3G{^fZulRTi_GGUGfczq!JS7}| zS(js1EtArVsl4}To-qNaucqbp+m3!SXxY8}CP+j=kaQc#_;uR@vcK?~Lq78%$ukDaINKP?&rNq7{2V?VG#zJ`#Fc{+Y3DD89togt@B6L(&xe1; z|9;h-!0n*){eTGHAoH`0u0iU)T*?ncVxDAZUB`8q|@73YH+(+pKe@|?KOU5 zbP-=c+<1FA``ahowx4`qcS#8{SLd%v=BD!{vB=v z%JD#Gg5|4eMfsfi$*eZYvhvDrf9%|3w0=kJ9X#7()NNPt@YTIjE7q6xHJdM@*LEYP z;Hn*UMD0Tav@Zr&HDhmNc6mniZM5q4AP+#+`pVIgNdJ!W)#=9ka4&z68D;r=WU1xl ze*fX;b$|WTbzX57M_CO~8mmZvQNRSc7Y87^KI!~FM#*K3Mh=B?4n}xmMali~@$8>i za-QL8ls)!b9W8JhvL$K){!U)~GP-r2b8}iBdpRVd!yq#kWA=CDN0I(ppna5Kc%qUb z@IG13e#{S6Fs&98km~Y@(@Yh!nlpFzMota@Ba>TA$=?-+yTBgr$kj8L4Ua?S6k*iu(GP!b-+Ddtim}yMM44&+DvYQe2 z$_$c@TNJcX9=uuV+GHACTR2hdUE-=jE#Udf`1Eas{eTs>0xyWtuFwiF(?(mAO`b4w z$rK(cQwAk%i*=?UnO0V^5Xgm|6xx?%*08>Uvs!(Fn7P@tfxC_^gB4Rp+rYQVnN@Hk z&?xgfsVQDM^2C&hEV7gW4i^``mit{J$OnEK}8qRj0&WE}~BdW}@LYk(7iAusY#55DMC?CXpf zNJ1LCT+(NmL1r>>auOzX8|z*&Et=z#E||8^aB!?@Xa0wV*}fr&roQNSw^ycI~B{8*tHcS-p$+x+b-l zolLAGqhTp@nyDP;ObD`8D&9OMB!y`eHzhd?q2^6| z`Vwl}AF;BnO<6Vz%3r=nT_Fg3yJc5akSxFu4Y1LcCUQC-;eTy`cxCkExG z6uKxp4zsFGDDy$ndBITe$^HU+G5O|pG1dqSoo9Q`bBz`Y*vQV;qSx61ALbmdkd$}^ zwFwx_G=YLVuQRnSyqW@t6j=f|D8jN;EBy&`l;5Z0gIAB0PsKZ#ta8;Hn znRET!R*yyhS#mk|V&a}~{*!M3DAGq!Ga%k#*#*3ZjOsq1O)9V*o=@c;ck3Y(#*0sy z&^!)lJ8M)IDH9uoJdHIfmp0N(chh&CKSj*`riK%|7)tO9P-QG@E+fZIW8VNUu$pc2 z5il-?IUE^|$GR9}n3|BsNSHco$|i}57Dmd2lt-9a;J?e3RK&JHsUC_+wBTnLH(3S% zsT0C+LM{gm<%)^)T=K(Ofio?Jj2 zH8H!%B*($jo__RgGWnQXX>wZmL8&Tv5(S`cdwC&W<_v5`&A}ln!vHoz&S|>GgZ~h zX)y)Nq!k<-UXZ8v{(6Oy5iZy|h_Qg^r00c-VdF99qxg2mqleA`c_Rni0-3e1&4|Tb ztruu;T&BrCDMPfMHh^>dFZ8WYOm_Y>`9ua8Lv_`H#ih`RFzaPm6tW+akv)Z(#voQK zSRHJvhlr)WO~YKMRbrPjF+@^^hflcIq>cEkY{jB5#nY3>C(qNSh^tAHza1&`|42UD z)wHLV9$ZqyD%moLk6URv67Yt~ITM}N;%kK#tOjcfsEm8@XeZMU=zwn&Yhi!1$9u?z z;zU&CLv8-(Syue&`DLV3RA|$On?7rP75()e{>gy{W{-2v@wNFjIAfu^bp5_-k5LQ# zne|%kEMI1r)T=S`^TGods_D^Fd*O$he14a-zPbTf8}LSw%y&jzw0571J%Q= z0(>5Nxu%U@L@%QsPOb#oBY&LUhUY`k&H%#T=qFT`Lx-?+FM|F$FwHloZhIxh#g)Nv z#Gg=InpQs5zb)dcP5+}}WCoB2NCzIoW|i+RrrL1^Y0`^_cH29fb;8=PRZS74PcuRR z`QO$U0FEQdNIT#-68aB+e!q916m^kaM4zJbiFo&yi1&G{cerwT z4bY2m4>Y0zttPn=Z*>Br(gP(tRTiBM^80=*bL>t!DtCT!y8H0wEfj0ks+f8i=_V9? zZ84tci!Z*oRIJ#ceK4h$(GJ1GTChz3f+hdG?H;05wz(U*Y??{ey55en7a0O7ZoT%4 zD$GK8qXPuap!(CHHfiWX8NR*?`I zCWnuFfQOO$0CWsNCL|!rF2mc&#on!M@robtzvv@at^khQv`2jSXK%ks;^&uv1wx!h zFi8XqpquSk{g*%epa1*azujbiuOJM_^v+}9UgxDO;+T%g34{oooC!sT4Q~Jj1VqC~ zBYT!5O;ie$LOpW$yWwz6%AB32xe!ZQX5uH40XI#+NuvbC_6xG)fp=5eBqagcFD~vn zyAJD~Lx~%Iae=8joYo0q3stcc*z&L`r5hlBYTRP2QJxm$^UIiR4V@Lkpd#qa+*(ou zQIoOT>^?>=&4!aZd< zXtQFh1(?In$I1ysh{0Mc5XSn`#oCDFzaZJ(ji4G7N>hkimWXo>YH_r34p|L8=};$m z!9@YOJG$#}%u%4TSY-v&)&dvmknvpt_$@sS|CC3bjW(DQA#^vaFCG zA^$mmSwQgi3py~eWi?sCNyNebJ`=x~x5RL`Tca$E8S;1>Zbm)?hMtXZE^#5#no&-k z;zZp4G+W5lx-@11ToTvpIqTlFfP5B>QE*aR8vvv@_J&j4T~}g_1-zrX3a)j+>)xI^ z&o6*{9+xQX_x;^Oyd$qr`A(y7#*qoPl#9|~(%R*$su%oDkBKPK0_w?J6zG`-#A@KO zRCp@xvP_1CHc}HK)64lnLb$h!rHleWJ|29`yoe!aK!KdGVyHk(Hm$jd#za#tX!(pT zpEZ)Do6uxcqrUS9m!`=IX0&jQlr@1sH62{%5SK)@T>7A5e)}a3j7!YHh$(hSj_C#{ z!t=e%*^JgC*Oc7yM&X)(Gj(x8be9pFPbskM*NC;r>xDAyi3~uFBH;yATH^_kPSzYs zKTFi+C<)`T(Fwp0MW;3I2cy4Xtz;?DJocWRYZA-W$FZ>XD2rtmig97(W{jpD$-|+4 zoG*wD><2lTgDQngC>9f&@xx|du4Z1>Sw14GWNb=`4EyXZG$~jI35$2rul-Zvys0^M z92BjfXej?b*N2Wc9H9T1McJDfnW{0-4l|AEX3Ee;cxgXa&~c%;3U}ULj@i(sgg^$S zRmLhQ6M$=wH)61HcH<#j7hCBUQ%KSxPMK?bqT`Uu@^8qPZ}+SF{^-eRvRsQP$kS@& zkoI^Z)Xp!EZ0~)wY?dZN8UkSTeE^4Ako> zn37BS^^DIvRaF#1_|8e0(+WUgS_resad5ngTW~%L^whToNC&Z?5CT5^PZMAW#Eo)R zgBe*R#OMX1%=y?)d5Tlj$03Z0N4AOi0+16esndS9_A(4|fY(fWq84ZG-x=4p?$BY!^4qH)#}eIr|j0lc$vH0zjbrF}%u4jeZ zgc-TrRgg6>hYuo~c0F54B}_|QK(knL2Hu?@V^UZ4?JB2PX1IhLVdYolHw+za*-;5o3v=pb`fGZ4{nN*xc` z^oK(j5?OSd`n%hEntPm749=Y*h{Z}s(G^-#TaP4&(**)@7t`}edD!DzO*Jw|B;W=h z!(L7-u{i@|A%?&R)ko-lWew_rt>c?+4L2&>q=3Q!*Y4;R>=$L+fMSdGnvsJ^8p!Cg zKhv%d>bEAcXQt$a?E0V#09&07HH!w!K$&ojvYdTP*^jQhO%ZF=#{vpq7!!L{F1s|z zD7QXspM#U0<1_osm=A9f6s2)Bf~EXqj~DBd-&sq5ePUh?$*Q23*y_I9gct~TCbF7~ zz2;Rt(n|JMGM8_~328mHLS~hm$>$TVaJ#D({00 zKGQ4GqkZQS*NHd>Aw@N|eA}*N{W)5J?gpcF{TRL6-t0g89DUhEUw`;#=~8qp$2<7+ zLD{2iRRt|D9DI5DvD;jOoIO0*KWTEwJqj7zetpyiU&;G{KHzEel{g}+Ui(=h8}CRL zG~^bka^Jxy8Mg91E~EDwd6Kl3k#Q;G?hyFQRqGbD`|VcQ4BS9XChv4y%CxkZ@XEhs zU1aVp>ngsC-fVW=ewe3C!>e0f+da|Mn+*UqGM~RQafT`+ zs*jJly8KJa#L{%H-CSq+R=(f0rQq$oqID}HfatpHEzc6MAfMYf*pz*Z&EdoUyv4;P z^%pUc-G23`f32R6BlUVM)9mr;l^ox9;s^IIf+#}OkXz@X>~a-CNJ~+O8=o9SqJRj> z#yBMMwBp24Fw^F`YmsLO^IK1niU~yC#G4|HQFnA%3mHku0O>+-*O8|sZ)%c?rzPL-d-dS1&Gnt^;JG zY#AT)z_kWNU2`?!S$w$$VP8cPE~peBZ`+g%EFIGx@Sd1A)1r1Y{99%}oJF2%m=Hv0 zKA~Z(D%Bb#2hA>m{+piQt3IH_PBZy^(88@y#m;74$4A4(Ci>*P?8yqF5;8^n+ZMDAS}I>Lg7J?tTXPPRXH5_+AwqAg~tg zdY*7C;M5x}5oM#DQw}f}$}4kdOw;kDR-rYBWvppW*m<4FG8Olc?hh}?T~1=CA8>v0 zF);GS2!PX6Z&$XIOCXS`{*RU03)y}11@;~95Mw%W@mdSJ*RJQ_A*9XQncc2Q)nqIX zhhhlJ#a3m`rCN9sPuQPAA(AbpJB!E^Pz|MZci#ox;i44Rhi2u)g?~U<$c(04W13!( z-B}j@Lc%ccPoS)Z^}==IvIbzuprt*_(t@2oBm3_p4ugR~3K^)132(-*dx2yG$C#au z;99`Kq*6oUN&?RjLkp~67}}F5?LK3gBZm$+SCUV|89CTq8B()R2hz2u%Vr_cE^p|D zF^AI6l8wAoC88jUWIe zb%NF7)68{TW$8jq;Yj_G$sQ(q5Q7q?VbTl~OI+J}Czj$?;~>^b+AuBaB#B>%%}y`< zAXDnzr-LgoI4d&4Zr5YN7OD`sUl>h9kL zm{3-T)kuB$X)s}cA}JCEM6TNXwNIbJDyoSez~|<&EHA)fOJ<9`N7{hxogAD=mN5g% zAEyQ<@mw%Jq&15RpRWBJN}vct`;r{^{<`$zUt)gFP@#@4OHK z<{%g0PNY%9YQVZVzeWF()si>GVqUlRdSheMK32FXp(LukSYj@dr$ z8v;eflt|8tbxk1I!^eu4Yd(vg7Y(avpTpTw*=5U=Y}-xyW54mV2v}FNc4q}!sVwh` zhwyg7;g>=th>f?TE{!=szA~#_bRglGXf1#;U;&~Ac3c}du_j`2T-W{ezTF3^3CQg* z@7&twSW`Keaz@(D@*%Q#Q?RAcxb2P6Erl#HTaiASB`;fvZ?ag}Yz;76H0MkeN&E$< z5Hk&@l?J9V)f(oD)2f=#U;!FF!=t1rbRPLAnP`FlS<#RwJr&Rq!v@O5gbG{_*B;>k zse+sp+rwf=0GXn_?!If+mM}$&lBsljblbIw6$!BM;cHP<|ItrZJyzB!@LtQAYj@F1>1tS4}jgYbj zDF+kibhE5~g^9V4vypc~WeUIdi%^@y9Q$w=#p_O) zF~+3=cKZ6mhw5hMB(!U`ft_ ztl`JUyUt@%fR|^0nk@j&Rsm{!#-RM=e{+{J@$h+>9d;f@l>j!F&=wuY+7gI7*2ERK z9ei~(3gQdSGvBG_;PEXWy<+;u_Bx=BWao+tmYW;`RZx;9I0HC!@4@6rg2*C8>?UMH ziJZM4Y$e>ir}~pwlP?c`+i?wH zx2P7-`2n^P1Kh;%t!pv#jvi(p^D{0f5j1C-sA|^Zn#o7TN*}^1cf(@ws4Pl@qH~-L|K#@me%s?VY`^XfM0Z2s98$cTDx(o?t;m zEq3vJ7dRlG%}S4&;W=Xvn8j>r-Jk3-su(I-c{`uhk_JV-uyIZ7eUdrQrflUM7B0rbwe4s+!HYM?VJeo6J_Dxb9Z+mTcM3PBIwjHoH;7%)fH zH|-9N>ur0}Ieb~*PAxLZN54kKKv{lc>2x$&^})3aNe#5$0cE@d9Sb1X>o+uAo$lOCq-uA z9^Vcd$mi@^!;3wQ0Bw&}uKlB;h&)XBRo2xpevm%tNYV_b+2p`?alXtLycXZfPIZz zb@XAXj3Q6#6`+GH=I|o=1G?bRIMV*%cz@}@Jnl6hj}y>(dz*EJXtHEj7#24NJ|Cpj z7u)TC5Q~i5>u}YPBJ?V8avYRZ04$x?xMlV78A`v>|lTjQap+JGq4n_J=>q-@j~SMf8|z zz^Z`{aBu%DZ?T2ixdRfel|jddqhVJ-SVI%IXc%F>i(X+~$AgDv%Fo1&^HKN8rk|mjFe{F95+H9m+B25zHrZrR3P!&I7=}Um%t>-~ZP0 zYQRI_;Mx?R1*YO=sgH7yxCRg*pujOcmAMI^#LHP*7M7n~2Vwm}Nxb*>K|hia>T=~T z#&t~&y4VlyY6jvQs^bedx(pJoTt6~_iquFWF%mJLR}}}>EYN0ee8y7iVA!t=AiJur z(}E&Y6Rb&v|Ho-6oceYhLhekse%b~NkqW9`3z&pJmsx?~fEkARKwezs;@h(1vYzN3 zaFkDPS?fyOoF*(2}|K zhMa4pZZ9jSPjDb55JCh45&OXI0Q3yaN}Lden?sI@7L}g;z5#-$v@)H+Z7ot+Hx!4R zAnOunEmNpr5}OCsgD5q&23nJGs;QXC935+rE~puek}(xv59c3iitCdRSkSii%c>}g zG^UWe4pB&B6eaMj3kvkkjCO6(GGTm%IvsXhrW8eyhn??+oP-P}nQ^H@$pxXK^IJ?b zJ0OTqa4)X?Rk71bdX1GisjiKdjgh%{CguY5LyjJsd>^iZDTB_{37|8FqA^$VngbWm zSTMesVrT+wgY7DY%r9WA5>zbE5}1k-fsz$gE=8zDAWx1mkCcm(bxR2{IC4DkDS7BU ziY<{bqeF6QuAgfn$7oKg9bQ{NXo3?vWm8)j8$5hTjvD4xo} zWHvBI49*OXN)p?Jt0$StqJ#uJ4~^f_p^{x>3(7i{jL&5K${D-BZEWljur5lnVAf++ zER&_NAoelS-a(L;4)M(p5J6=eGZO|xwuY?-d_k#u;fS0dSWv@Kv&wqdYh+n0xMMSM zLNsB4xV4Ly<0W!(N$$PyMN?%pLx^XT2PtZWs6J%E#Gwi?>@nSRUN$YQWTZ0+$B%qj zsIw_))!q*WWmvJUmJmoJr_W*-)&!NV3ndU{*QSNmE0NtSZi8JHqm4x)W@(TVtCnff zEVkmBBq zKtN2UGAR>21wGhKyiD1-XdL$)7=edC03xmuYi}Kg&9z<6#UvtB=uB=0;#UciZOC+Y z*FmYk71ZZE+X-N)Dfy=THk^|v16u&FhkQ>`Bbz`r=7_cgTUo=b;NBLOn?r_|xktx7 zirLbzkqLVsFiccMvryRQqV(1R#BpT3VaP|rBdHX~M(cS(Wg%u_#c_%i5(N|X4p@Th z%+8@43TxqG(t_WvQJ%>bL@R0?w8rJ$aScwv{tAgyVG z+l@bmYyjC=I;z${HkKtxu;YDq<6WUdthtH_AF$@HZ4Ra2prBO+gz?cQ;GnZyi($bE zfF4qk$_x8G*#E zt0%#+mI+)ATTqj8^8VstFwZTCWeZ@MHw!EICNbA*DUfoCw2&FV3{5xP!G-P=(IHj~ zdy(p#`4*_Vphr#k{^AB}nCzC3R}u;(3&^IOOAJ_(RL&pT+?B!`ezVsgk@07WpIKpo z3`-{eAKSu6*^lHNH9{E-@(z^;F_f=M)@IpsVk-skgq(Oy#hGIjigPV04NQ}rI>OOo zn4cvrRQb}wG^1WY^RuB-9QjRa5X%g&$mD{@-gaQ^8sEJX!$o$VkIQ{P6p?4S>ND;~ z3?YD7 zW!-T3%x2i0ko5Am67K{8ujf{L$cDORnanbIe{#wCjhG7oPf-=|0u#-Q+9iXk2E`2s zP$8Co%td!IwY6AowBX@Qg-gH#Q?fG2yv#IKz*9`CEaV-n6#yD6nmvQY$U)h_dR-g< z&-4O9HRj8518(j_4Eaepys1q2?s4ZyWn`plE|v3N*EK>qpiJml@*bv@0}w{lP^1RB z^1eZmLF$NCe770L1|HS|)8{6r)Wkp{i3W5>eX;9=icKl-1q4h?~(HNpp1{)?p1rY?M^?wh(2|6HM?$r^oonp2Qrqk%(E$? zMBf}%?LPX0*aI@+pYA{WeDmSwy<#VZ9bV`+{YE~^UT%OS_lJMHQK&=T-fFMD12`r6 z42lm*SYes^H#eu%x%!7=jFG!*C>CpAL~eNIpq*^iP+xWFbNp4lVW&6iZtT5GN(KTT z#-ozQkrhJ))K_E?f5hYo8!oBQY z(RT3nalby^_xqbGoT0ddYbnO$^cG=A!UqYT^-Si*sV7F=Mjr4etwtSf8H_A*{6%go zE~BJ!|As8gJ!D-lo3*ZF?;apO7SH-HKu<$2sHV&8p6QDwva+M-0HhkQAks@4-o`y} zFU>I-4;}2j*3U~fi1v~64y*fNSi6VIv;g)^3xsTHU=99mtGw*MJNOR&Ah$d!0O@)^ zyp#K$c21WBU-AfSRC^L%ZqW5~^4(Pk$*tT<;WkdQ#En-3^9bQL(_{hPQB&40Wa&)R zWHvj;u%tK`d@6g{F9BVM{=F&rIG~bOPdSZ)H33X1bCWFc;po1!{6H2mO3ndOIa7o2 z6V2_N(=XvBqRqH_>FdEJ?9)S760N}|t8tQe+>5M27a(vHWT$JuwkB`v0-C3N_xB)Gk{2LZ$m!U-xH$F>DNv;;gagjc z&i;wRnTVgT9MLrfpjf31=_VDCeU}FN4AUaF z;dJnIYp@u#g(*N}2ZU=9H)TU!M}VC`jU^_lY=m#5aU#AjW#;@yp=64EInOBtd10o0 zu~e?|2jwiCy|YgZ=~puJLZ(@-C#*qkkO+WYin!rC`V^C-7akFXK3FK*TEYmPIeI2;au)BHqc6bm zLu<_)M%ExNvIQn*XQH?!C5ls+$~WZNu{$!M|1?uwr*&PiGV26jQKMXR0VQLj1*BfV zNT3mj0;Pm|8gd$6Dla z)==~6$nAXOFB!(+cnD?V9Gj@KxspeV~(6Qv@V!FHX9A)tBOY~S|K|fC#gqA z$29`8{zvrR6q5t7++U2uLT7^~To@47CRgAM?IXs0E*o}LP)|o1!U8ln40&@7xNDKc z3*70`M4z$UZ*{rxvKBVvXksOj5yA zF@bXCh^ZWqFcu#fV;Zhi4Td2)y~i|bjwCDDD2G-KP2ZqaZ28i*V;VUqFX!V3p0{1k z#4909I9(1wD+%m!F@ZAXT1kZ%W6c5&|Hjgas1p^d_xxoh_%6`!QM%V`4A(-&L$)ZQ z8W~1TC;2k8iJ#@Za-9bY^huVy&S8bnnHhX~Ym+b>>LDl&B~@HzWZcelqohe5lgk-c z38B9LaK@qm9n_DIL!eOd>3|yEt%p<VQQUT)usv$C9rW!t=%&M6H!C)qh?8D^T2c&0;QVX6kJ5<$q97~M^ z4Hq3u(M`2{3qLAgI*P2wEi5H;Kg-<9m$wC;9TMxYONvCM`%TumtK9`J1Bf zhtb$Gep_i1Cp@{~s@r%};v6wJcaq&tiw39~uAv&krM_L9@vv#=#x($Pw4`{3jhkx% z!Cm%%ocCdmfgDaocfo1t$3@AFDMkb-n(%p2F@#uC@>)jM!3DvWQ!g+(?ah^g3{kPb z64wV3np(`^C3!usKLm^dJ5duvDa5R*jKT{ea8*v>1=RAC7fAN7bV1k6d9D5~a`vTM z7(ZjlK;Z$GiqGKe;CT^`jdq0z^jMmtC94ujCk=H|*76hzh;*kL-&HZ*n}!U!efNG? z`(B8FT+#AInVGv@Af7-OO7Y6HSO!N`Loc?7B5PFf_ zxCOiD^{~J9ijhfy+8RPqy9h|&Qpp>2b{EX)5jv9t$3utIcGz#k0<>wF1BXay=^5%0 zX9G9@d1=^X)blT(I()V{$p9v!6Jo@MCYkI2Tn;e7bj#u`E3O7~IXG*P!5b%3gXfHZ zo>9T^?tN=wb0}7lbDs=4)J6Nng|LBa0z{R&?gNcZEC$N?0{yWar;5012^0JK!UwsW zfP?dVDCa0LXza@W`5Uk+fVd!J$v7Chj1z}e#{|_T`#U1mv_0>KfTC8|RK*OD*2Y7T zfa5HQ0QDSa)PSFtrqZP1DX!s~QO(!Gz7I?;H5io^ysw!$lS&UVQ?UA1uIfSd`?}@{ zGY3)vKwB_d?=cXeqgoS`!DMWvu?ZR>K7o=Q_7?3rUWWJLo4pTu$>V{&A{)t+?L55; z3X?f@KEr3BprK^21~VzblFur3#D~ppIQZ`%PE^j~!;>7GHIwn7Gr+AndsxwkugoV+ z_hY~Il2PJEab5&&(qq8;6uEenIn^ah(f(MxsZ0$%KHm@Pu>>O4wA~W5u+R5p1^j{a=cgYcp2?3Obp*egLpyFV6jPWhF!A~BbC8tTnk(0zz z;lQ@x1I~JwRpN0qWX?5@LcY7zs<{>Vj1pr}zM`d^b9O(5FCFDln!#|>X(burD<$u3B?EqO5AHBPb z-hTJx6kY_{A|N6At>YNlpTKGKyY0*q$xwfkYCD5L2|n2ml90922d zy+hpuiQ?$l+hP5v|CTTIg?akj1OVk8B=7s-=gBMB-><%FSL^n1b|Cv4jPz(z zH8$UoXWzO1I7DxU=gLde-Z=o?v(MkW_K$z_-FNbjSEmC?x)0HJ8jHoBU%%zQe)08p zZ(e@#e5PF{-ffQ~px>?gt(?O1fqCX$FLb3lMqhSET^y6{+u(n$9Jwo*%HzKx{8I;2 z`Yv;5yLwc7m%eGYf8VbHoN6{_91u+wwV$`RO0K#Ox;b?4crG#9aRe#N(YSy8i=U!MqibxW_ih7p$`M=J#zM#CIjau*NaIKqFX{kD zY`5E!sz4qQ#g5+h2k;}gX)<9^n{c{K?`;DjpYhnOuRf7YVg@u6ll{`~LbCxQjHWes z5M%b{0eiKqUzzuVlIQDiVGM`UoddM4PWwH&>!Ss}Bc{!)Oj~&INy#q$CU!{mcXBd` z#mpG4i;EQm7@&41{{JsX$RWSMrNSYP1uPra!2A3wbnKALr5yUvL4I;^FA8!6@5Omt zKXG(QWx~;zlU=M6)|PJFTV;6z!Lwfott3z!sF{`m5!BNYCr?+z3uS4B7DKR)ydFz@ zE3j@sxM2!;L!NCZ``>!7T2`^3?-{U8CPl)qv3oZJAntA`Z5%K(4mL*7kf0Ix3{k=a>^f$eOHLAho)7-Gr;fKpdT`){(Tyc}j$R(Ap#mM4?#R?y|T7F*zgN0|Wh% z{G1{l8dNmqpa#-nAB~2(Hc)Y9weBhK31~Ae$DQB-6$n<8Tu(L?^Am?YrTTrd-mC@C z9}9KVKKIk4S(&iM;^Y1@H<*8STC$@NJw{;Y;cN}knoCP%j< zYXab?bCjbq^wE_xB=ui`Jcmg;IK-tAespNak~V!T*%?#bna$2{El{VmK$kWF2viw- zdoC!QAb8{!dCaKN8Hg$=>jfOPAKjn)=z>$XWc2XpELgL&&NvHo?^CX*wdD-EPX?Dt zG{SlDomZR7%8JY2rb(kxCL1RoC%P2i{bjxS(S5N-Rm}>DGXPBq8V6PrI<)={c>{%- z^CdHk-2c)fVTp~43qxm#qlt21-Z>uaI5m$feKA~5`=0>ETsd!U}?DTkwX=et-f&^^^hRo ztV{YT(~Pw=iQ|mU`DyMx2YV4im_~?G4dfA-0dBDyCQMA_ia{o4$lx4Q`W#4~M_^pb zQbFh}X=Yc;yoeXLPi^!H=!eo5o@Jl%94c`OEU)Tl+PGvW4`L1P^MrDqfmnx=ya87Q zu_{XD%h;B;NeTHAF9NW^SQcySpVg=M~=N@-U! z(Dbn>Wr(Z>)$0%SpCqR|;JEn>*Z6aru$*>8Lk08ePY@qzHH5TAqmU$pp>><&a<^5SAEFhNlCW8XYvYV0??5v2^2}mu2ut7U6{xqOA-< zXN+l)S?#Wae1#U1k5;)R5Y$a6;1NW#vbaFKJ39o9^)ziz#P`oQFMmkaqC42&P^N@OJ^i%F^R%uUr= zBy;qPG!U)CLpfhg>No@pSq0b)@7ZC`myMy|f*?&YyX&Boj_H`X2H0&K^wQ)YV+VZ< zdRr^ZFrEM-@rL;5Y0N^zWK;o9k}VYfbSQJ5+)gG<%~IK&2bC27cz;x6P<4R0#A1fH zNC1yq8s3}UtDA{`ST3;M^zZ#nlt2`n^*v`YDNs0)8A1FoLy)WkVyxmSx-$kDv5)|h zqmQY^oV5Te44u9cAUXhFGOQnm9N{|i`c2_N1z4$QCftVvglh#uV4mZK#yVzBkj@8Qy+)iAlk$%2|nsx#=x!dDLx0A$Bh`1EDJE=NSYiAkU<2cL78M1 ziF4WqbeHumbG$_Di}C7%qF1WS=Ik|x78Se?3hK@BRMxOU^&2A=rht>GN(u%R-n9MN zH;7T8&JeGVJ9W*Z4KNlcZA@H1;+~I} zKn?{XzF`xmUG<~Jn>0#mMmo>Yzi|P1P}cftH`n4(dD}8BlB!I&AUUvH$}K8j2hVRa z^azV24(J#IX3KrQJB&nP_>?UowHvQlb}P$ZdxK_r_tkXpCwKj>>-0O~zrO}@ z9{=%eciZhXGOSW|{c)Uu-wmg|{r%I^-n!Ic?@@}gA9eWGk(iP1w9(%M+o;7fGeM#t~hgzyxxz=29KAi&le$C>9fhK?L7> z2BIcy7%CbJ0stjSP^h!t;&@-uEY1pMLGBI0P{UJP&QS+)odhJJ=jZDPfJrdO%nCf_ z{gWIxaniHs?dRg#E#xkInjG9aMZrXu7Z>~9RH?w8n~=ja-4k51tccl|yLKnd69Xuu zF82{@6EU-(!qMN}Q zhy~A!a>RSrxmLK7VeU*W*95SHdU@XZ40jC?II@FvX6beL*dQ|axVKj|P()+l9&;cn z(DW?KTz`9TpnWV?4NoCsgtzASgLaXu?en{Dr!avMfqD+G?>sXy*k+o9*8g>HS7Qt_ zr$^V*XE3?Qj6+7=fiW%^V!Ird8FL0tbEj3(q&cnbDSr>RauVYQ77O=U= z%kZn7{N)9zrIk7k+MuW#ae#k;wx_6b>_EIQZ+(JUw(){^eWv)C0tK#wg}ZYeiM7ab z4r)&5V1Vr8(8B7SU9P|d!!fh1hc{Fi7YJQk+)Mxk3s!?&OB1f>0U*#V$|BPW>xI&J z51YtKS}>_aiDhOOK)!1|UjitJ`|Om2I6&U z1PI{z3;DL+2Wk}Af+!~!VqC6GB~94*SqFlI z6r7_4v1%H9O9|(QNHx6k?%Id}#|{!g9CkTx7Gx8ri6v=*TD*@Iz-sj{r(ot0Z8_Q- zGLxng&RWT8Ps5QXdJ zbI=8f$QLxo-Q}WK;NQ7E?fr3^6baOEm(57mLRQ=YhvigA2+FYyD-VXyKj?6B=5JKS zZfgMllz>cUj17rQ!-9cKGX*HX8Caf2qqSa62xc=oiKY^iL<_KzT>TLYit# z`An^t_Iv;y2RWM#&SFjEs889bo+g&#R8|6)3fYMGHe$q3y0+h-Xs!)8;kc14i&cVx zNUM0w^6^;44m#0d!+3>`2CHW2Ogi5GaBAQA1Tr);+1MX}PfAw6lCsy``mQ&79tA8Z zFTNept$}z0hwQ?%cUP+tVHdW9P9swWpUaNxx&a}?d4pk8x+$A@A%YMR0=`vNvEyqr<@lC4=#rDT{%9W6{_8*e z7ye(GS$%RnC{*o^hRa zj-tQ*)Bo_?=V4c^O8`329+xa#Ssk|5K=}joUfOMha+r;ibP4cqv`a5#bH6N$I11D{ zNYMSJ-JVcSp30O~T@px>bgkADhnitApZ@ZvcbhbNmM4Gt)8U7|{AtraKdassi&pmS zt6BG~WIUw5ez)~GJN!9=evoFur8fuUudK)5c9{HJ4NLq(`k20mTrNyW5vr+-$(X^q zhQ`PwzpOWSdWbgd_rrcl8*$rjF?g3=a0b~uEMd?y%YQlR?J~M;WhX;@HhO;7%1(md z!{dwSRkvzShaiozb{VW_4+aJ6JjzYl*~I3rS_)X9m7ZJemu%hpkP}b(sjpZ z-QV52gnfUEoDAQnfHW<~{}K&Hwuo_HCW0jM(tSU-bU6G_Ez0Fwf=0vr=v({4gUvRz z>Ux=)ADecv5kR-JYI5n^eR~Zo1@+eHK%RAl~p!Ur?@+<-JPZj#=iTQWw+_SZ~OH_D~~h5a3etI_gCW1Y!Awg zaDTd1t^wn4co?EtFdVJX?(l0xEr8AB20~RQajK)2?%Gtl30O|_)_yj{dEnXKU>(@c zUPRw0F{TswiQ~&K1W%P@LON`O3+{@0y{|s}ENk-8_4yDL;-X@Ua^D&9B;EE7iGRh# zP01Xb-csnG0)B|l!8DN2{;T1z8Xlsr(C&r2u`HaM6Ub6-{2kL=K99ejjdgT526d`& zL-X69Z|o^+e$(z+zrxZoU3?LpDoOETvgr+H=%RznYw?y8VHCcssYvMy+D;UHIc(SN zTdxA!%Gg<@-7qoc0ynFyCN1RDnsb>?=ZChAp*!?U8x)(-ZOE#7wZP9OBH>vp;>)$LFlnd zTaV^zNH?3WPKOUagWBes{Q>L6{{2#aDxhr{r(LF^uSy#z-)@e*iwz?1HbvLYE7NzX5o} z4L6jeFlksv5 z;wt)9zu4r;WO;cGvrGBEb*_(-o0ie<-pEbE91?3{v-7Al$B?Evlj8?|BkTMz`ps!= zGxWDISmXB&RCCL zy@tgl@7JHCmu+`w_ZYq}Pe&A@ZcKo4*h?3lJ;h&-ed~{XcddW=+#l>t-xVSUIz8k* z_VZwg6r-OIy>vU;HndY4EG@;#_{2?f{Nh_EF@rvts_d-_}*3vDWwj)PtiFa2I@#Rr)JPoGcRBP_bnHpsty6xT>Uw(1?pI_`=S}$9ZS;0O$VasIkX?Ke&0>4#-M@J!Cg@dvxRTey8`*IQ ze5`m@n)SK*kMI({Yyn0#Roa75Fv5_QU(Ubbb_;{a7KU5kozo5X zZ~TpHUSI6-Em!ho0*jDk8=288?#ydZHWAM)l@yo_dccomuGvYb1!Tb5c_pI0SL;ow~&HG1u|K_taP`+@8CJKLT7+7XCaI7dEy$gk+V)uK=aCg~o?^WG-14R#AhwV|#bv?yoiEeiHz5UgO@0H`dl&#Bm=?^%8r6caG z--wapU4$?K;plYTKj8_cU91aUEslplHfuZAwzH!OKsuwW$#ssS$fQ+1y^q#xU%HUGwXxogPEreA=pZve*!6wceg=TLnPy1p7>! zMwy@(;Op*oFd=3)D8p)XLQ-Cy2=`mvL?E;sx2y}si7n|C(Rn2FD>!K4avUGkGIGQA z$0wg_(bKJ6IsEa7Y`kvp{`f?Ga)kctzQaV(eZpx&>(kfo;kW@{N|gRVGN>Q z>i6g{jM5K3A+Sl=Ft;+K!D?)G*j>W}aXXRQ@IzR-sVjFs8~@KyTByIgJnx&9Y@ObP z?Z>|SGuffkH`5-V%h(jHYb^o+8Y`N1(nreaNX2b^B1+c{0;1>pe82T61#gMm-D~SF zVgG&Qw#b)vDk&dn&0h2$zWDv}{tnGj&P3h#Jz94OPbhO4dGPAxxCglP?cq;h!BN^{ z-KuLcu_`n`05t`l*Yls(2QlPhX+YDCph>UK7l@ji35t}Xo6+M_(3YkvbPRs~#TS>d zq004@{`fn0GGEWnhRaCtZO37F=eClw(oy{F>j445Pw>HBdx;qI);U|&+aIOPW-OF@ z8A9>u#;u+0aeP2TQ9G+9ku$^sH#MXy#Z9^CkSN-3p5lYzg4@rXcdj;vOo?-YF^TER z%9b#GC>PZ<^JkEm_P+DSg~s8`7b;Afoi+pd)4xc%cRr@RyBi||ytGe0tgGX_U z4$sGtw>XMy2djQ3AK(rshwYqO-Yx$lElY)UMK4cxr^E4daLLvidr3<(NC#lu>6+Ia z`DWrL-N6Yz+I#og_w7Lg!@^@QqwA}o*%;=!zr#ncd)tF#A;WmgNso%hfIeQ z|J+AGrYUkPi?AC9IA4wqRHh8Tw(~LjeeMiMJ zq_N|5KT91aAky+1jf+{WQ9kPSW!gM1LoU@F1V}4~V_~_)(xFl+m$XvKFW7InF zfcz$mjz2yT!{+$MC-#XZVZqO3zr^qVBbMva=nw6Fn#MNUUT=>8gS^$1D=d~Fdv)61 z$*=K-@(ds(bWS9Y1@&&ib*F}azm2%wd)-{kc=SOFbv-`sw&KYAUO%jRRs@L`@50(U z`ALGWo2T)=Q*nu1??fY7{H=+)TWkj!kR2535WU>5)Ya*)k#-rExr>DA^meg2cl?Ep zY0uq~5nBjF+<8_mZCK^;{>uAe)>9cv-FClqw!(Hguvne@VS9HEr4Jt)X%X}hYxT>n zy*t^LwEft2@HDqwFiMIljlMg|EC%XwSj!TAkS*bp7RQs53c|>EYg1N@DxHiO%ePJ^ zO;@H)rFVLixaSVDQNfIzWDv6&1U&AtaS4({af{i z4gQI(7@t*sVRx|)dO7>p-q3@NA{ws5esX&?C{@m2CB503>i_t)A}0SHyal>h%^&5B zqkqdDfW-JNcI`l|J( zmBEFOAsd5{W4!gZ(i#cqgBp%2gIky+>%=4UB0VlR2sFWtnhxK>#FW2?#~rK- zF$RajogFb@@%Zm3Z|N40j+>36<%0YD%8)&m7Fyopd*I-m43gj7+)PIrdf$sC2Ted{w7@E)%GyNMI8>nyzDQ@lY7rxt^hry;!~ zE4|%>2%z3Xr$ZRUZWe#WPJY_qKj2YMJc&<1M2ji#UUt9-91+2-Qp5Si1siaLs*|qo zy|bzN`=G~m={i`a8s2w?<5o6h6o~FWW&tn`ZwgMhT`QFS-iKG+2yZ_8EC=XSNIBbr zc-4Wd3VXoQAN9qpkt)*+k8gEw?w$V)qX2YZ{ zOv$k}%ZNYBTX1h6OULgeo@46p5qs@3Ctvj{wDY7s3d3>wTDHZxJ>E~i4}ZRM`@p|H zA^dq|xQpq&@q5|oKm7A%wwLJ6A;-edhd?Tp>lb2CuOXUa^D3#2_m-FrYt#5pyx%YL zLq6jKC+G%GfrNRm9Wk3@%&4*$|5Vkn%pbSX{EJ89Jf{rt%v*3@0pSI?>xX}oH^JU0 zr{vlV4)~wtt6eF`WyA`A4=jW|+7teCXHmYFVr);c2j2hnmVbDty)!m(tS`jNP0UL_ zJ7oDH4X$mK{s~DsHz*F^Vw{JQgKvtc36R5p2}Ne=$K853*!hLhnBg$U|E$KN1TI=FLvl_L`dwlbt5sbaoA9#<0qVU zC-*^aF-dMu`)#j(QAGIZN0^IuAe%!DD@xPo8(CfM9hz!pFnX?~gPqpn2#Mb#4jyFF zzCUetcC;clr`gWW+Wl5+iS68IJvcxKaK2UD_yWsZ(xb~|RDYwes>k(v*t)N9_B|@TF)7?g*{85G^!Hzk&?)Wyt+^d7C_eG)c~p~)WCl7Y%Vak~{&M9>)ZXcg{^5jx zhs$3)B6O`i^2r^UTX*u%^>CaZ2EX+&$KQ5h8183Swn`2XhRY*#^`!2<4x4(alqcOCEtXPHOp_;_1ub`P{N?U$?tq-N}<) z$^F5N-1+6R)6l29*QYvrUk^wDPqMsoi2WG2zL8Pexv!JhIXW@q#--~rCC*{f?qa3M zspik4q;x;VF&pYm5bkn9cu{HB@thif=I|=B;HO_IW3+FAu5$U>9V&TN$H+kG44MQ3NY_823ADsX_P&0>1pZjBD`={GXbevtiC$p*ia!Gif2y%s<8 z2msvKy!7BQ96;nb!rFSd54wI+EZ11K@7nvwqn_;Q0#1}jO|aA!y)*db^ptVG8s~!x zD!IHU_Nd8J(c`<$Vogq|a<^_g7Q)fSVpE2TN)rkq~l&g5;7jGEz5myqx`<;J>pTb_4EGVw;Jq|Vfm<2 zs=H=Tzm*q2!O?CvIP8aCF*Y?dX>u7jGvA!K_rw4cgX1NB>kB=^6+?MWW~cfJSt1Klt9Bsn$xX< zq_NCM_wcxApw%PQL<6-38WuFFxZS!EgoZ|UR6f^7)R$fn30iHxIiH*oHUvp zJCFTbdu{*lC#-~+eijEm0|YglBw%ujPj-4maxwFxF{gV&K6$`|pluyotEGwzNNnU> ze`XH9CYYR8bzUyYT?deQs_I{X4+UQHZ)C~$24|;-fBC0>L1)7K)N~t*_$+4e#Drg6 z)Tl*!A^uo(&!Dh{uFz2WmV0G9uO=UJM0IaBaFT;!10Z}cmZhOc?cBJ@5|NcR8lC0I zZ}?tQW}0%%v`j4p1?zno54o@G zWjg`mTjmHK*!2}zoj7?OUYDs$72D6>JQY`9`q2ALQGERQ8@D(uXDVizzTu9*?fH%b zu%WUsI>N}>)djL=tR%%MT?S`o=bk%(ClS7P=-7?y!%ZDD&3*5Ad2=IHd)xq2!)&F}AnIstfm=Lj^Z6@yt+EaJ6&$`STgVG|+=79%nhRET8u#I-*s z4LfS>7Ul54w*N2}U!F~QX}`VAnRP`Bc<=EK3wYbVtsK#IP4k`Kc~#m|Fn-NpDQQ*| zu#}&QdXARW`+5lEu4&N?>wi!Uw09}a`54mZO}^y+Rs6sA#j+}EM=uT!EkrSlAZ`m$ zJbl1)S{>on%&ZwjK4M)ASq!t&!7=P1IYYYhq5^UQ!uOaAvTQ;EEhenJTGl1xRW7IB zD}V4B({hG%QmOBcXl*I)xP~;-(05qovSxt|;d1T(k4sTh8cRhCA>Z7|q-DL!%@~u% z0OT0)tqIGQEF?)sDpsA~?8~;j0v2Pfb_IhBJcg97a$KvNmNv63Yd9&JfNn}9D0h<9 z93)96fFai<(px0jfiN!SH*Q$Slu;-zi|>0Uhl~udX2jn-|0MsFcYL6#W73^wE-EIG zn+^~z^%x1Mwu)abgrjZCd%`;?>7P`chr4vqcX*luH5q_eaJMaq;_{ciJLI*BvELSA zJzWiLt(fvN*CzBinFPrVHqG zn$T`kyY^B%6aak1L~L6|twrI&M3_yqRs%X zny8ou3{}R0IYfO9bnsx9ot??Ji~E-Kc$5tW$4KnVMT=Qs3KmyWIJd>ZZ38A{rujqT zXZ9xqbDsQA-u3o=di*l4rXXO_Pl|d!;7JXmXHl3S9n21?3D!`tfbeo1FqCK-i$`5- zn#rxWi=V|YP6~W|#67O~IG=pfxZiO|>w?y=y*E8)-8F%R*jM$UqG9n2J2H(v9*KT? zMPhq7PvKh|hE-c$uk$s1OqiiB!6ypd{;k6Bd2}4KyM1Mb5Chn~DK%k;y!T?{T`X}X zFA}_1&{7&znnD^uo6!7S6$OkXcgH%RxAp-aoF5-HiT!hO*mRryuhY(RteiE&^~6A z5vd?P#tmGmC5n8-(csCqG4}REJY>3O@t+cT7;Z(EHgcoS#j@GgqjwtP#vD5mc}oj} z(s&jnND+t(=1MdFNlMGcfc(eV*gt?55K zT7Z=yQ_?#+w9=ey41#RkvOW2&)&e3ZP#N76llt*bV#x2kySUzGwT=NzU;$`E6poZ= zU&jf%nG~Xqm@~^)oebkPqt}4$(LQ4e+%ZiMFfDY!Et9LhuH167lG9y7=uczDQqk>y zuZTRnWD@xf3E1&`Q}lr|l1=J)ASDpBK*GjCaUsuOC~B_y z1lmHHap;3lD^jmbxIGAPetN9uGStssz?DbI+2j@mlh08Om|(au4`5C?po%#;M5?eE zo);-h=eEb*e`2>xe!pSKX(?85>f3~hF~$ouHXa^kHSeeABb_4_y1s$q(nrc`CH5ic zAgqXlK!>z{OOo4`5g(s2{ZTwL=Byn4`H$izot&Nj`Hwg2eHylz)hhsl5I~COC+D`K zPe;KGFf*8V4024Bi1#bnHk2yj@hl158dHmJMj?2>Q;vpVjCg66pB6p@^ihVbMK6|;WpV_ornm!>t=bl}(eS;t6Apnw@&0g*bi&_1f8X1C z|M;arLBF`kgpA7H+QWE1olBsO2no`HZ;T5@WoNU#&l#NURQ$a_+avK}J$9Ebt$a`fsp3|x9xR5% zUDkas9-H0HqlA%}LJmvqs3N66*uG{DCqgFYCo#0Yh&}PLY$nLMPoF#|m&h!Zk#$Hn z@YA!C{d5G1%0QIwO$q2SBM**}c(;-V#Hg_?1#W^W3i-eD{VYUL(p2cvQl>>t_v4?i z93HDlCgA91)s#uQ}#c>h^Uu*J5z8M~fsB|V5WwaSkK&$waw;7;2Q@yUW%g;Pt zR*(O>EkT>gdNivHWsXN-OZ_4wW8zKE_zaQ{8{wJZJXxrQIytC+Sm!{f6E&Sw(9!r8 zi@zK{VP&dv1*thw^SBrX0g%C4W7qV1g29Gy(JG2CR;GkXB@|A!0`;b$mj)dMvu3fR zA#7oC3%`ZWVL-@$W`);}SiIkyGu$v2yESBksk~)LMse5ZoW&#fwiE*+2Z2ZZvHJYb zhFy3lkk+i`1~T12>ta@i5^LQzOr_2fnk$XIm(G7NQ&Y`iIUZqVIozU}hRUxd9%|I@ zW(l6>jrVf1^5`(QEF=(H`~KwM)c2-|UCRO>0iSS}4Nh!34b0ZR+=9JxNN#B)W)h=Z zo;Y01VK6(oFP%J7FPJ>&d5FoO^6`m07@m@gkFLOGT`2>b(pVwH)TsI)HZ;M@_t2)L zCL&Sn*8-bkn(b<*`r?~lbN1n1O{o@pdmmo@wD0Uz4IPaZ1EJHC{pFwkZ4|iJ zRI@wDR)cB-Id$_8SBm;=j0qkc#toQEyI61+^ZwuG#WeSAa#82wR;9DTeOUqpG@ZNy ztF@Dag6^yXba@Jr$fZ{ZWW*9%Ijy(VJtH}?lcsr{mW8YdRq!Y%;N)r}6SX6gJOv2r z#@thGnvm$XgWW)`A>@>Wo2`B9S$G+ll~>oVUT48)90lh`AN)JI@)0I6IzEiio8i#f zh+*rd9BLQd_Dhk7)RO z3ytY6O3SU>+_0JxI4b*r8sV}RM91`ofBxg2|L;qCftTL#Re>*h>RXIZUiF5u>57fT zmLb3BSWvDczwM8WUqTk!-!3U3V|c9rSX@Z}+1RjALOhn=*l}VFR?^%r9D8~@^;fw7 z$Y4QBS^G8O5GJ|Z4?Y?Y5s-So94jv?z|1a-ISQVUrjY$g%1}!tj7Wml+Z&xyTI@5# zf(&xLP~+Ji6Xvd2p@@cwTam@6%>^9?X`Ne>bI^Oo4C6eZ09DRk@;%(hzoK7>;yV`I zprl+b939j57y!it-$Moqy30abL$GM2ly;uT47AZK0dLyWj54r5853)|A&TT+@KHw) zlJY3$_W6`1k7cIj4@^@!*7a{oC&#GxohGNni$TtnY+l?^T!gbzBTgM11>#gqF+n|v zjZX}bhvY^k)`vsAuvRwFd}_7f0jJt_8*I*=$o`@cgj^%j+CiIzw}e~)W1bQGAs?=} zy;5t>j4%NkbGuq#cnC#3`FLM8MIQx2rj9!4@ZMf5&l_tc8UjqApqLMJlG)QUpP<8p zSvI(anmG?SqMQg^Sz;%P5K(e=0!Bc+$LQdsHO(5-8y>pjKRoxD{9D!-(_qK)qm3b; z1)@!x0XDOoInN+pEM)X{GUqz!PwXzMzmmrfv(e$9{o?FAPKo06F82c1YJis=Gu2zVb zabR*%)^zK^@RH<}5x1?g7Lg*axQ8%BiQVDh$cB-=fAMJ|mY^Kq(V-3C>_i8kdDNo# z=xXcEkI9S zS#DZSLKYTTeFH!R4p5J%n-2HY*IpWhCO@2OZo;C}h)ZylVRnBe2AMk&=4AUgsH-!w zSstB+y`WGp{#PVwV-NcM2B9KlyVH5;?lF;Wu=A&&|4{q&FaPuZ>YIEx^80sD3QbWo z%JtoAP+RF4nj z6qHLlZK7V0=OA!VhXNsV1Mq_qyvq9m1t%6h6mQIXc%nS9BonURCG(nVIYclD$Mt5l zMocBYd!RE#(e)v*=RoLIv(6@&R=6Zb!-JD>9L^4cq|lH%)5YMaG&%#dI`KPMfZ+%x z0)Hxh4F(5?$&Uwzy=TS55Z7m?=QgLtH!;-lxK}#dt5zEXQS`y^!RP$=e zP<`vY24arn$dEDyx6~h(1hN#pAH-3g&niaiJW`l-k}pWwx>szZsqN^YMZ<>fNz;I9 zzjWCow@cSfzrL&1VXy1I@ly!;G#4ThFg)BcsCi#87!^ojwkNRsVkgC1I3Q~%JPAYE zcR#aG!lbni+N-nm@=|jObVN|eb29QrnVxMAM_aZBb{WIBP(6#hhXw6fx6KNHU2<=) zLb;$oewUyT&9kqV0=}TvJP=?IT3;N0^M8Lc*#*9h-Sld=l zkC_IgH3zDCLSt4@=F&fbZn$@l_!8(hz$#;k)7=n*o%2TM@eV=Y>FZoL2rL_3G= zA=sQDz|l%}IfHv79m^9d06;LuL0RU2wy*00287zAuuxaKwaOz}m^(`%Oz@#(k9*`7F@(bIcbqfoB^%t}=nsTv{ebD-tc{%2# ziccU(P}@Tl#8#KnTxG?{(pAa;1Xg}q1Tto1#J+z^43Xe7E4({0G?h%oAQO^L_ z-;`vg`H8I^-l6o%ra6D;|5P$M(!WA^0VyP!M(H0+M$;djPBltpvtqiRkwMSsv;)na zAS*nic{Vumu-f1#iCt;7>zM2i+Ipv(IXgeXTYi5d5>_L#V-b82WHG>^GDM8*=M zW*E(ej6%LEPSq!yAZ8zkB~3(yh1DB)DT(qo@xYOKD{l}15+MnWnu)q!^Ldy>OSaB) zDy)npOzsgA-POwF7R$R*POHkotuuC&qiM#+-n%)8pHoK0C^}l^{JI-8w0Bbs*zC5I z=+ssl-NjjI)?asJC*!4{WWt5jt`fIug;rSc>DF8+BvRJ)5Qdu^C_kMQ>+iI8!b#i+ z2#7VA5o~#RsFqZX(@P@5NqU42oWOe4O``_BVoegu4?Nov(><{kA{8QOwO7)(j~yU_ z6K1<8JowU!yHia%JUkDIGMJDa5T2q)r&h3|?>X5F!ROS&l&dw0s2eKDri9eSd|TX9 zWEEu9TriUGBj~Ks=q|~P)xUR>itl8#0Sdh-2|SvxBq}rV9T=5Wns0B}J>bG)Kiuwv zuid56c`blnpW#oHD#_QZckyXW()w6$?nm&ZDdOJERvU&}8Q5!s;UL=n9t7VtK_@3) z`JQ+R1|Ns<3JZqn)J4}9j5Qubd%K;&XM~{LqAxie!7X%CHGcV|*@XxokF5=-m~5aX z4c~{ZOr#&uF1mH2wo^yphh}Vn2{eH32|Q8zE%(|Uy_Ze)Rael2GBHDn>GuHu@9^xv z6QDOK7obj6aSjG2ymc9VWKe8W8_kw5ZC*w+gc%2=1*DTaE=)qu^{w){&0L=;0%U1n z;H}#m8#e-ttTk;q?{)>weeJZ8KGIIXdsrzi^UdVe-fpv6fn-msR@VgL$RZ+k%9>nL z?atA<>Al{3l8p+E7fbjsoE(aWl$A8O-B!S5_nn`jFwO14t!48uMISOWi?f9cyClGD zvuJ;A-7l=aR#ft~P%h1ep{qIz8^cG*g1`9gnU_!@wyw-nnna77Q9C9~mo6NZ*zayr zueUZxV#QpNJc=2qrb#N>TQ~5b{!X+4XM*)4SwL~!mBl>(Kc#jQWI&vpRcOV~WF446 z`VPQ6H*3JwzrxPyqjYEQ@egYx%k{}D9LH47iy59fy~CYCfryC&FCJ<_-eK3fb*ycB zLwRFH@FBlv27ovCiZ3%8M1B1omH8{=R@tR&+v&S(SL6q!NjE^8C)9JIovUuV&0GOP zk#QyZm+5`XAeiU(s%#Q0)wB?8hHPGpu6A4p+?p{>}i=3gk3G1vMsG%eNFqZ>Qlj5}?- zwmW?N)qZ2G;kI7a_q7(WuWnJc)_I=l-B-4T>gidZu!r9DdZSxydmR52ozgztmSZ(a z^v4l^@;7%(^PQk|C#-gNxG$y`-!5Va0&>r?21Z-B~VpQ{prtm3Iuea60EOCK@&kma* zyxClrPh}K-A;-PuMxk9%PKqsJ2A!!FD8GIxhE+agIG|~tCZz;Wt`7xIy?*?%`h`X+ z(U-Km@6jkb0}YUyMoPVkpRU3JgoN@O%(qqA+*r$W?+OvY^Y%Mf1=1vEWoxyT9r2*o z+>|Wm+Yp#Hu5!mm7V?8?6$AZY+u5yP!Fp`4Srt@1SVc~)Ec4o`Dop9HFfRQh-G?r) zt-gZs7Xkt`m?v)5jD@7{Wjc9jPCzIq+r{?BZ63e$_AcLFi%F-#9q_AUltJ_`7)n*a7GD-eTLQ7Ww-2H3BlSWeP%utjl5UBw&B^Geh3>wjB)o~=^Mfa|JO{%pOqH&16)Ql?U zR&wS^1?&692ECTc#q%Mv$6E5wNCIhj7Wv9BbIt>SF@oX?5$LIl#j>VQg=Y<8uGRb2 z2sf84my&Oi6vDxXatuI0JkMN_Q#QbU4P!Fa?_fwlN*JP*M6?eiIcP#v9^t3*fSGoa zj&vV2!sccI=*+TR0a+?2&e*-F&C3}3XSgREs+T-$609i2(yD}$|0v>=$+vORu7WYw zgWfy*Rncj>ubgh1u8bEi0(Nr+}#5>=j$~?nqLKTIPYl*cbg-E${d9H-@8G%OZk4-~-+|zO%-= z6}@RtABTb`E+gEh7Ws8MV%nKJ(mH#COfGkf?=hN0b)Yg=pYMo3Qx1u{A&%|F#T}g* z#&gJ#pf-%PRdx$N@eXtb8O3dLRx{KGT5Wu#3#M|Zub5ErpJq$-B|=T+tp;FI2@x~G zJnqaeO&GM(=?cZ8K+XZct(B=sO{x|@BI@FAiTej%eydp3+dH3*K+l1Hdo>L;C*VM_ zZ8QmvOnWDVbv?5ubu`MTvUcJrrSiF}%FyaZ!}g)937rPGVX$_snZ#i@pe z;dk9~<)cupEHf}TOYd*X7WE)mNR>}t=_~o85BF7{ilG&0lx;hZH}YIYvJ|u{;mKAy z3j~XR?Iv-j%ocMn#DELHl*@{g%%N>l0MkSZhBHc_z_L-!p&$eX=Vyl@z>9bd`_{mv zfx1E>GMu^L!HDb8RLj2nUq3w4M+N=|5gVgXI%Sa>H(u`q1BP!(ZTd2VD>Se2BNMvwrb<<@sRenfLGsAQd`le2X=j3=E&f{kHTgSyyZ?q-v z41QoGFbT-fe0D85KRD?o#{8jnGSwz~=s`fY<70|nN{1E76t=F7g>Ns4aqX$YbLFq#C%Hi29a9LVEF;b^sfo6XUe_#_9-p0s320m&odpy;bM~6~!3>8K z61d<(?-rQ*sRhw3IBA!m#EKJ_q=x7%<|RDi*#4>$*0u5&+6_Tepa@r4EF3?1LaO^2 zt-iW@4=pE-@t?SG8eiR&IOQDJ?MZa$AR1D%gK)vY(Z*jI=oFHJjpvZNr(DyBg7LCe z^1_m|xIN~UH%p~P5Is!7kP4HlJ-TkSwpcu|h1IU-lomgK^Q<)6Np1dGsTtVy8hwjl zF821GO#P}57mh`tPxh^d+*{{ZDZ4u@z%it_5mQ(vXWzB?gYtB0)n8;s4qxfKg)nyK z!KavQMM`7^+I{Xkg4Z5Hn$a#T6#q-_*y`@_^EYkLEHk786Q{nKEG--(KSJ-p&2PPz zT<(pjNxBf5!dn+uanQ7@S~FoQXLrlu6GLbCT;%XCj;737Mf7&I$tnMzX_xO5%6IMj?iJG z+dGLQY+sIMP3VW9=2e!uid9IAtJgA^c*kEelZ|tb{J^%OjJl zIwMlQB=)}_qo%hv({?FvI~HR$YD{d&^pKKH%ni~*g)Uo;4>6bOzRv>r#tE{uIH1l~ zCHkU8PT>^!Xpy=w8nwzRi&relPb_y=^pKw0o0NJTi9-$Qq1}?V*?@*NZ`fH2wMXeg zXa)=jniAS0?q$c(`be`JjvXmc{Ts?W@y;fGRO9> zeO~v5$-hrz?Ohj5f1dY$tmb9W+xri91sO4{+Nqha;VDiF9c6PP+>Vb2%wuJ;FQVjx zKM1xr6GA}-i3kdPeq}iF587A>{BgH{){TuZ06xVKGogrz!#C0~cnHD3Bcm=cXDP{V{{ipJPKOlBiHRwv*kRvI^;>`a1fzLSbt8PY0g?8+e@mJvQCXu;AXX$ zEYdCW`9?dI>Gt{gn<;FzWh{V}UEe}wsW-sZ8Wc5nfo7E(nIm06rZ(l0136UMIgoK) zF|@K^NJJtzS`K)mY>T8csNF-)qeI8TMBNdwB~}Kz87`S$-6%AZE~Pnvc&>Va>sSmU zQp<{945~%y>s}q;RVvvmyWe7$OVq8Mxm9JFb6#nW!z5r}3o%s?e{(C_wvVDoMm~_w zqJhK)b&J%%^t6Rl!+FE=P_~j(6SWZ;-eL1;QO}XKd6Q4;Qfs*1lkb~AT(p}Z(YHst z_fEE0a`w^3r)Z1ko5D9-ODNWP|hJz*`B$co^%J zt?V3RIXnysM4DS8BSQPwnYAfD2L!^j&RNElbPkEZ-PmI^x+j5(apqe`(K7g zaJ-ix)xntx)>E8Z0&vZruw;h=ie`ij`#I)5h8?WM4{op+2Gv>1!lf3i(UVO>%@C#f zsv+EPb2Hy)tdW}5m5x&|9)#dVvTi&k%JW7Z(h$A5&0#lug&aR6v)dw&^bk*AjvNe+ zv*FQ+kwl&=bL8;oFgqCzjE`;@a+ z58LjUM*qz_098P$zYQ8sLW6#Iuq=()sij8|E|@-nT-TmkNqO7P-`s4L6Gth?y?*{C z>-=2n7OiN(9f^FKx+UZMx z%Y@llcW%q%a3N|4ykrrcQG4*DQVN>p6=j#{qu* z%5h&>zXkO&W)C2)5V>}It`~mT)F0uUo)5C&`Kf+}Rt+fbV`jiF7@f)8&*Eb*fM=WJ zPFy5oJdR&plTzs_pFk_e6CH)GM#*F(=ZD$x;ZZU}zc?8u!_(~W^z3h#j3b%)(dhhd znT?Zkx$>+%9V2A7M~A;K9|IAHqm$9sCPZ|wc-DVyLR7@9qhOIqmVN)Umd)41J*ZPG zr)Y5AX;%=gXeXW8`)g0Tb-K>5*un5HJ32Y}%Ck~Z!PA_v&?Z_9Wii<$PgCNK#jDSZFv%O-dS;07J-bG_rSEJV9ZO}00Moe3a*uH{I z0ze1SGj~PcS8-UpY<24|E{00ADv6nAE*ojDy;m>ZU18saX|yUHyAb$GGODD-2XZn8 zX@+`BivHr3>uJ#Y#<~)qgFGl_auiGeE$=heR+aj4B1?Ba_^bD{t`#D8!VhzR-`FZ4 zXQliw{pQEw9*0iZ!_qIDAIX-%%fBwZ5_oaP>WH_Bj;&2|N33GkW;>6B9R$6A_UEbw z*DD%D!#~jXGi}+0X<4fbwB@QR87Rl{L2Po7n+Ls%We=2u zQ~c66Sa*?3-#sxFmD_i}XYOsLgFFz45-wcPjBY{&r%tts#nDk47BH4?kt{U|v1vKw z0n%F-h9xYP0Xl{zU`0j1VPH{opc8hSpo@ItTspP}wch_^mn6?x6u+G-cS2G)5VjN8 z`MXy4bRD~O+_4S%%d)roW@J7dp0R5|){;|%-z}5$Fq#;(zKdBq$yNfI%sIZi6(v-hqQ~>lZgghzR)8(Wroz$dJwshTrPaFE?l{z6o@Xn>7m3;G^xG z*mJfo8rYkMi}x|ldduxob!gw@vzke~*p~4_cS}N79?MpD@hN*RXIrIWv6@#8g^C8* z=Z6trHar-H>$S^zkbiot#4}kv{ySTXJ>h*yBE-hnKgcYE%$*Sdniy5_)+qZoqU*GU zBDyvVDU1AtWv~X!cKrLx3mO5Q>kV*n0^M)GR%+}H`ch@=n~8M~Z8KeCZBK|>_M^it z!f^NGMPSyT^9!V~cj~wHya2ynX;EtktEFXL;Bf9vtuD;Y6DlX5g^g?tcAdbH_6>Ay zw@yw_km~y#08=3N@p_dl)tDfy*h>g44{xj0Dx@sI*s?C>jY&W;FgLZ@n4vFAyT|-Z z&~%ZXAldF#VegZW4FBbgG&nlS(7`m>KYox#QTeg| z*Yp>pYO-c>QCyUcL^U*zxr@$Y5Dr|{dkf;C`9oiBtYI)d=yicaeubXrh*z%_{lGAH zQzmT%9HHHv!(2;%k_m%pTp5-?FG|Zc9b?&lDuY37@&*XXS55Q49Pu0ZTAqMv9ygPe zE4p3=0)9r(j(31}aoCdj$8cl<4Ph}YbDQ3%52-hq*>Ab?2Jv54tJwW>vC3Js3C@&= zhj<}#jbQA{c{P?3De{9Du!R4kT~oVl9~|_q%TH;7|LdoFmnE1viJA3kNeR&Xj_n9& z?^yyJ(fFx7JuTbd+G+)se6wriWZ>XC4l|R<)O=K8na3|kUA%n!vP8yX&VVY}m-W<^ z%cKfLcVfDQw$xSG#3B`Mx|nd(6kezAiyk`17%S3W{`ucdPePj$JBf%q%NN7rQJ^`( zNe-erIq29Ag!#zr5Y$DciiIpOvfIi5=-85Qt~X(0+PkmigS^|-)m4xmXJA87#)yO7 zZ*I{;JMI;OzP3_2MhS({RA9(GXq8p8Mr00@D)sYh*Nrip_8mvVzj8nQ)z^#+_g`+6 z)OM=qw&u3HS$nF_?Q3%6p@ogeqfOCQjx-Z)ei3aY6O$q|K19>c3iIUWIL;kAYL8S& zA}h)XWbkUu|IdrMVtO>|73;~te!A(xjL?U5mhr7k*0UOf)|u^`v_$QxA#C`?^q_4Y z^E9_F!w63{>1GGKtCTFpfvhq~4^zCObzOUz*K$znZ0n$}6&c17bgkv=?mu5l#KIW5 zFyvL+R8JLY*PvBb^TG)wpcnM$tL6x;{<<*j{hz&8CvaD!}s%s2h z)=JK{kM8Ghvb-ii?#H;7z;%5)JQGbYmE)+gwR+<&Ec z)}_quY8}YMP6DDvWlHXZR4B&3?bf_)MMGmE(kN*=KS=O5iYoc-<9|T`vRf}_-2m)g zd!S0yYnhCERn@w4et-b{RRz4zwRw&&R2PSRE&6{mMuS)-6hk}0%}RL!<4f}a5btjN z>SZWAC3A;|Na6-lRW916Xf;9OizJk?eu{Qj={MS@X>_VKzYn-bj9xhP*_NlXb_`qJ zOGrg_!&ozqaz&}mzW%hvrc(^5WlLEo`0z%~@;m_iwb`vH=Ye@z?*92x)943i*Tm2$+V_ry zZqa<+?K~tb3t>gcyvf21(MK)--pa!7d`z-i7$0zJl5snF`sr1+NTdGpRPBJEy(gAfreaDTdkYA5x#jHv6A9M*Ccd3lOg~!i<**jADDEz!D z!IP0{9+45|35Wr=2>!X&gNr;^SjlL&8oaH4tYm^pXR^@(q_l)2ZIxvx@zZ1O_V#_k ziC9!=zpTU;Sc@}O!hROp9@vW+yN+|q&8o`l^~IGg^u#m?3+xUX4{SWa^hv~y?eeLg zN>=(8HiCK+rhS%JLqWvhaYoFIO^}PR~-P}#?o)(Ztr8!*lU{GaBhDwQ>RBJ&3hp09ICyd;wGEGbh$9q2Pmm zWeH8c;$kyj=Hgb%QS7s3FpMqt#?>5T8kumH*5L6iLG0&6og|(M&%j1cwHIy^4nP~d z5=xq4VEO%KIV&3tuWD>G>pqR&t?G)kB&I;dh1EvyHjrDpIRGphv}2ikM303+T{T*? zv=_xv=_mG=<)?iMS!b^H267t>s$MlDy`+#B_>IXX4-q6Kk;|6w3I$$@F-`P%x6osi z5?V$)oSVh*bhG16i!ZnIK)GlRH?iVu#MYNNZs($5K=u= zSpnCYxUoe}TGv>DEqQepMupp+2gO)NVDo; zYQ1*!Ez>){T6)>H)*+NzQ>O#lRj3^kJ-_@6Y}w5+aY|~_+LlApu^-`hgAWR`O_;Gb z{MKvplRv4;k9^ur6mgqTh*Yt;ODJPz)=>N2_z1vckkIOycu**L^3T8`ZoI^6E!yx~Aa`a3dk4oOSD}&yWB7$FRD0bWRZRKyFwoMhByKK?>8>aDt8g zrNN5LY`}b3jd8n3Swl`~MQj4!18eD|;Hu_MF)}*aeuo&!Sa|2X@q?Oy*e>NAj60Jq z0eL+b9t`8HEVAc!42e>Q(Z%V)rwW?J9@WeQA*~aKSFnOuTrf0}( zEBBq8z-jTR>jPTc9KH|>mCAaqhF<(#+Xq!+{``RCGg}+YiV&XW%h|{3Bl>3^oW|AM z*A8Y55cN_8x!d7+yvW6!xXYt+=<>5*Bg;@ayAFY{;lWX|pACJ47R~$}=_4Is6_dhN z#?dDs9HUQ~e(^qX>S2IQZB2A(k_@-vnX-u)Hxq3LAZscPaM%EKZv3>yl3Hsw|3(D( zMs~>Q!RhvN8r1taXpQE*4?Kr^#|Jx3N2k~PlXrEKKcL2Axa+ndj-dTDdRKCC%W1Wd zAMCigwfW@ywHAPx#bI`Md=e%shX2oxSoPG0Zy&z4NUg395+}eRAD#5T)e@}Pa}91e zb%RB~{1)3II8h6N)w2|k*2I7hyD01Na(vL+#?#gW4K&eMRUXg9hOI^)5}|9mrMJ#M zd4m_2!(#2BO-P8pCmE~LgJd$5U`R#SiGvG}$l5frT_9KTSX|u^M`orbje>j4gV^cv zuCX}2tXo~<4EF~LUVc6?UJiTorDltD5rGD9FzVdItfFAhsved)TeN7PanuPKk zT4KN(c(29cm;8QWY-4#!XJ_*0=WjgQaZ#_qB=$=7NlAN5pQU&s!XvaC*UJ*zike2Q zURv(;ddmW62X2y=Yy}u;klTbBPYl4>#A^Z_$0EG`zU^jK$mC75m-J#iy-53mY0FY6yuu)!#hKp8ZYht@am`dx@BX%`?R~>$&pcrT}<-n_Z=>lkJ0zrN`3;Wr3e9|P+Z3GYtwSyixd}1 zDQQL{`vEXE8p+wJ6|Zk5x9H%BTFE&WT}G`kCd*)HyJIcHz}{OFEkiwU;n3x&*`G1R zk_o9OX*P4T3z%(CQmH$v ze<6ROOV9gaS(SAR6o%LhDKRapWBR5eNr<^!>lxCwHf$kp_Lv}sA$xKGW>y?% z`CSI7)_aQ@br54;KP<-8oS(nQH+THUe6Fa}XE&gE#%?^=BOmd_xs zYr5dpmZ5Vl9li|yi=~4qVjAQIjm33iyY3SmLQ-#&IIyb1-N0>H1E$p=3S+%TKCP;z zjHJV0Y6PVkO=vwJhC4pMOLA~NcUQ90&B@ce$=$?W2*!v|yue zPUmZR>*~JwD7S6GZ7*-@(n$E<%PfI(90Pq{R_wMQC$p@pecQN@L^z?jJg}Op$@EeN zDstvx76ihz%C;DMUv9B0-B+HAif*$S&4Dz)OzGf94zp1E+;Jb_922g3nc$(f(7h z+>m?hO*vNvwT&SRZ6SNF4FW-PRJ(X-NKkz@?zQ${T2eZ^y1H%&`a1e;?hcwq@Df~G zNJ*P#kZBxC!FY1%Q#m`|qf7~_rnK%BG4W!`ebGV3#s6>Mu`^CT0QoGOcTlh#Wjz`gx$W>m8+ zcEZFRlByO@wjG*gh}ViMdi+Afq+QsUT{M3ZLbe&A6h}(T-#$x+`)NbO)`>`0vx&>; zzV8bVm-1+-Irr11T(1!bI5{8SNgZ!?)BF2PUiO}fWh9mhIu$a1#(T0VwYofW-}?-Jke8}G4Cez5eHUaI-&7AG)*t_QrsZ*$ z^Ud1P8?}>v~ zSLLVFz5qyqEDUiAI_9hKxm1ZXEgK`3U$>4GcG5+?z`y02*D`~zmNM<7Cl$NVmoP4y zg{)U^mb=*brG!tEZb$Zj074N zTVkLs4qqU?k$cxaP@7oi2Pd!RihmSkeG5}x=60!G!8>^ygB%^XCD5fkje99y*P**k z!02P0l2&aN(662DIPv7$P_y1%LgS*p(hJluG>g`L^E79+q4?gz@GvQra${ZH!{1X| zt2Sag^jLu&cAy6anQ&gu$s-nfcjZRHClE#%Zg1oy-3^+tJ`v}9T3XT+#^RQr3sfjVr zueFhOJcx1YCDfhA5gRVMoErkXV@O6j`${^bhF!E)P2c+&()@9g!Jc(^=8^h&`PTM(rI>_(Z7^ZLXMAbZl9%$+UOj@Slee+{8s& zbPJUwHfeg#O#w4WiH?F`Koq=DIZaz{U4SyM35X$hM@FQ^yP3smKDdqAChii{GNL9~ zNWzNcg1B({Pk7&xv%>>m09-!5OiL-!{IpGJ0Ju2(T~A3+8rPUz@%1JkLM}EjBN;o< zIjNwH+eX~>gWie%S}ptSt@tFM7BNjq`+{i?{pS3HvrTWZjlS<}qBqqHfzj`SMCfk$ zm-*v=7L&K1_DD#TDz<5C5uTJYu!bUf0!~fQRX*cMm(On z9b*e&i50q8*O3gCeG@dcdhWdtbhwrWge#HP7RMWQ>aFDRMzO%sL-26hBqhc zl(}NfZFO(yV-=qP&B7FHkGWl==c{ODJ}qjojB4v)rspbzJBNp1%e4&prOlHT@?re7 zIO0vQx@81J=CnTOy@oZLb>1mjQ_o-?iLdZHOe_OP^8!{{SzvqKX7J;YdCl)28h_kO zAD|quXTvJIgb{Q@2H<>}Y`{yT@v9(&lP6Qw*qS>@p)PxBKP5W=1KY42#^o)n>C)c( z10L`m9C)4c8hkc80`cI!mj9Jgv5d?R+m_SGZa+UdiRbUK&ObkhF@ zP5x`Kr|R4@Umt?P8B>eY<rYfi*;a=9b>T@ku9YKb+A`uf_r=Z zu`DkvpZat8`7_A8AT3yN>e2NO0vsHj9=iOdZ`O*#%NIXZqVku$ABu;ioTvk9*&>%O z({$U|V`;HiDvbi`qZO~I5IO79YJvMSfbXE~u?(cI!s@;*DJ`&atEyNxDN234_Jhv- z>!z2+kE14>huQathBFk1%&bgi<`X<{UQA)*w0b;u?qR7CN`qEg9?d+nv1tr9i(Nct zAo#Kp@1bN2^mUF91-L~D@IlN=2%D_;KA-Rt+5a_8IBMu}<~E&)eu7GgSwxA15PMS9 z7?>hG24Zs?(^Y|mBvN1j#@n(*?S0XJuDnU8!D>$qB-z+Wegn2v%3{XozxOJ}R%ORe z)f|%AV(5L3vBXfhIwu*%xs%2F( z%4WXgLF#SFdqLRzfJPBxW1xRq!3B7xl=jWa`f?$@y{zuqtqJsi86R8xnd^^efVX|z zCki%~w}96#N(0Wxm}N8G6Tqu62@zD!!j8CeD$7bPN+ zfTn#fJGUl>wq>y{^TkF5te5!whPjd7$dtZ%MeTE4E-H_!k7|*~bjaV2 z|C6i{daJ_@z*&;R^}7mzpA{LSYJI^AF#1-#UpLq=SG2Cw*uGa%RNGe!a=fj^Jh|m( zt!oX+0I~*V*+=H|@@3`4S{~HAO_!1-7y(AIDkrIcW-i502kEqjt;i+=fBC0>QM&yO z*=Vl6mwUmcuvU=x7pvo=px5>UXDtc=JtloIA!i%5Ig2{mLNrPPV0f626^^}w1CqHlkR{jc^!xCQrscR*_4oddeK ztb=6+o_WRMvd3IZM3$38R%il#m-B0L`uNO{--k>u+{ekUE{dLKj?-NF1& zk_^YL2WC=jACOi__3-~94r{bRHlI^XHFN0L1EN?P9HfyaVH3!0#^OA0YHzKj@_(be z?fyI?dy*YxlKh~TD0#)L+2tzHUS+<4P}EUEFN%pOr_HPw^dh%Gx^8Hr6s+?6!|iUk zlm^~5*6-oxZxHNuO0K|d#?sVAlL8h%M6h9A%q&nz=R={$th&l#e9=RD#q@!swD96u zp>QRvt_>{2^VnV9aJmhU)#v+bIiz)AtrMq}vifPhLN-AZ-e!)1Sry=-*2>jNCQ~NN zJFCaZzCGhjuzMn2Wfq3R6orRFYU(CGC1SoFdJ`<6xzV(M0St3cPB37Tpe z9(?f0jgv=mZA_8V1{#6}6Og?5p3AKL;B)?!cB&=Rog{PADW zbnNxxmkQ|;F6}5qk)+HKTgLz3*7~!KM+Jv{55SbbK zn#COSv}|6guNXO`Hwt~b`xa>{Ja8Ry^y(GC&JroQOizC+CznhZVm00kW2s%2KWe~c zr8=&vS(<`v#)oCF;Rs__&pFmf`VM9gFw>^77+FL@2DsI;Vl0-jEc5fa{3tUoXO1`W zK*M{nhvy>U%^5NrYo9ceVeDBjh7vf-VwUX%m$Ljn6sSmHzP8s-C;N%i`#fOIki zU)BlOG^knK*<#!Gk;_3>7;5`w{outR3jPf9u8|Hh=vk0qOa8UNs&X2waPjQyP@Ffk zmymxY2b9hd^mdHP5gY&-ghoG969t%!8aXoyfe^WfvObRttG_s|`67Gh`V448(;v^- zlQ^HP%9ZwrVN+&pbDxXgwU_r!Pu0!ZY|Fb6YP$FMM-kjeUfgc#=|g{p|L#XHWbjbO zc?1S(?Nd~2SZYH9b2_yqSk40SoA!oAwT+Gc|LnsH4pl%&WTOa6TxM{yy0R9G0d958zZJuw+V{MqZFg#jWY(DkHRjn3dl85+M zldKdX2fYvQ7p!|}iN4s=&BVb!-1bgmO3@bqB(FCNvOfi0SDuzH0fe0{utKEFeTCpM{71&YH%GNb^(vM2;RGREFq&8<;l*#uG=OEcGq9uVPQ z&F8kf*W18lS%FHFSCO3T7F|XjoG`jneoPVxm%5T7XqM<)VYEuYf~bptA>cYQ|I$vQ zy%nARreZ@-uEh0b$q3f#oT^mQsaG4v_CBpPqDj}-_({uxvDhlXidOmj3 zp4}a$GU-Q*?1lX<>=4eQ-%a)uVlqqj{9u#5GrOc?gm&=dvTUblM>Gk{DaRo+F*qcJ z!E#9!(!35LPRZ?o&kMnxX6=J(5nc7PgRl=)JfcSHVuLh95~kH!W;5=#!;s9Q zg{)N7>^3owmoQ8aeA>i-{i0lg#3B;sT*^u=@(6MkloS_yD@y;(4%?u{o+Nvrx3m=5o$Q&@A^3E#Ad-A9#0qO)R=^gLA+k#i+r&UiQ0 zxW)cQ_7i=H_r<4>Jrv8S_Ste^5$VghtNOEwN1-MfC~sQUjHjVyTIxD6{#K5{=LSVE z@+3L?t%&RoBH$nHt7_Vx70W^yR7eBDNFb4Y8gVmru)#i5UvdU`PZ9?c(hXyO$PXuzn>PMiAc3B+KSHe6>57v{+;hU z8NTlnf4D#6x8=0#X*2EOQ@$WvJoJGfJfEf@Ub~6#6qA(5#_LsLE+HO*DRTF{!?ZKm zpxm1&(~x?!7}(i%x+ml!J5cN{f6r}>ppe}=GX{%-rmg%81RHgsh4~z1y}m(&B<9MI zDxPmX!D5ZKrf&R@-w`5sv7F5nGLs~kPtQ*LM2gQ3YL#d_N01BlP^g+ffE|!*QAA9s zML(gEx#SDpY$(M%ITzm4+c^3M(BdixetP^ug2x>Q8}vUYl13psE?)b_q;;CgPv>lOCI*bP|3h76ZUV-<%t zhRe@c53seJ)qg>a{szTA;lSq_R>F$T_|3YCgN`ew>T7Tg-u)bQxmV&CNv3}26sEMe zGWRjYbAcbZiLXNl0baIRNgK4h%|(=|W@D^?QfDi-Nu}`{W&#^a30adCIq3+x6c%lS z>9`c>Uy=?)&JV^YOa8uqDZycSuV*LmfwyXeaHff^9j$TefTc9TCDXPXBr`*rUSIV# zXT49F8l0lSO{b{FTGk?L#K99Eh8u!?cYAAENr{^NwANmWygz3}u_y_v3FuDf| z72a2isKKKml*!!eDCbp`qW3!Jy{+!!ggNcCp0Lbhu)aK&L9C3A&)iHZ0 z>JhWL72U|qzT*fy44L8q-M#e@85*HY7!*~+v(iq^B_gu(R=X>I9I3Hz z#vf-zF=d%V$roh7^?&T$*dW>iO2L$|8BE6Pf#i+F8-GQ!IrJc-p5G3nAJAW#%Ki zRU%f5X2a#VfdKlN|9K6eFf;V0eq?f@|FmY@?1070MXiU(U($$>4=$$Fe1&WwCYOj%M^RVHC4C{srohH7 z2xR2r`Y-db2+w)%)tFRhrrPdMeQ7Qgus99SK9k$racYQNN4LpI4J@2#{q0G#YDzwk zx-5(PW|gm$v^byTpXoQ!`em1_-0AInYrm$Poib|Ddr{8vy2=2o^-k#goCG-Y-n+2^ zl;r*R@nudDC@A9m@jovC*hWSR3Yg#Pi59U9z>sD#-)#yoD#H1A{KG_CW7l?X^0n-s zy{$eta`AgnGHg2Y-}OE`ei3g}wl$7cv7txdj{p40;~##%0o-A4S2q;JIf!N?k!qne z=+~rLo6O;6Mkc5GYMT!xomihU_r84EJS?t0<*FZ`#;?EKb864QuS5 zl0_Ab26Z}Za=dP*Q6j1k4mukh`&{;hN9NRp(n%c)5_n=z_SBk+@Uxf?wn#_h=d%ti z-c>P1bsRDuU-BW&oS3kyc?DuG1dqS^w6q%jh0*!px)6oGV283MfERRH8qr*;52yZ^ ziH!pf5k*(mr0M7o4fXHJ;-0SBnzAX0UVV5m$Sl`nP=no_eSCs5|JNwvah(76;ge&L z=qI~R)I|*u_UtU{{YK1_w;$x0Z{Ppe17R^!(m2iR&pIcbL*qks+oP=a^u?Rrho5vZ zUTz4{sJ+0tajLl_{-$yNq`t+d)ZXXQvI?&r_PX`+tNHD?uJY-GM9Z;Bk9QiheWVt6 zvO$v#BKK4Fz!pNK{cN@f38p!#h4p1@B*_?pRUWosMA4YACV@0Lo@u$K$Hv1uxoW*^ zM{XO{oP-wEHOy(tk6|(<|W7Cp+u@wnk76o36>sFx>5ZjBS&vW>RW>Ku! z(72qbYv^czuRNFA%MRv^X>=JkXsSr&A$qGuA~ zsEBi2R!iW+3iWDZ8@rqN!-{=`@658+NA#jsFm^i7wd_1+mxwrQdj}Y<{#r^VO3)ef zNR&Fcb8?XGQw~FmPz_#QsOq8KXjH3M#@VwN5ZjgqPZ?rc@D=X5EtE&@8L7TZjRuzO zTUZ0=l?p}ybvI)I{5?%Jh9jN6t1$X^Q4^ry_S`uHwEODgLd@jvdp{!8!Fd!{m?asD zit+CkkN;`yi(>#~ky_-w#~9fX^{se_y{8aX$R#oTyvTd`24VG54b~0rb;UwWEE^WJ zCf++>RHjeZ)nIAG`~F;D^^j?Bf(~OJNfKd+dbAS#IFm_`juBecaG5{zM%g4TWpTD(j}$u8`V(g%>EIjNX-a0AyW>tpTw#({kdimeuc*6KTz3>vi+J9plym zotuG2Re3_fF$4Bt$^)a>^G=b%`KR7!) zOox$c5SgVo1_s@rvTubhk>f z+L1I?V&RGHbV*zfWg?J;zOvM@+4GF{@B(DRbypxTizwd)iyv? zkJ%B#U&?0QBXJuAxiL5xr$2Ijb{U$&{XCTDv^CrcHRUJA@J7}MMNMN!$jdp%chwN= zQ|*w0G3q(p0lRZCU`sCMuzcx zP1XAfo*@aloR46~9D0tGc2zPfSKzG?<-XC}T9jo4sFtTJwFhHp2}XYZQMgKvY2=`M zL*KR~_o(*@z@45I%>-6gv*T8-auHr=T3v+FauB7CH>9_DcAj_8&X(001vI?G2baQi zdoK2KQ+0z`K(!=_0EUh%0BHo6ucicz4hMpA?Lq>vr_9; zJtM(-A8<5#PGbG2G;+QWIbmnL7q4DiSe4}EW<|m820S0eO14H8xXOYZ5pyb#6%Bn-al!kxX;OvXwuDJc+IyCrKVC%>BB^7y1lAp2!5Vo6 z3p`{FqVQsb&&!*#P%qAW0eUr`>h)>-T7+>39SsV0`C!M*?iAfsMt8`m1gwx)@&3B$MxkPA%tun;pdVl#M z=?>Bz?v4z9)p+z;fgywQ;Sqm3Jd}T2(^@sYmWk)~iLRzoRZi1wDTiQqczBlC?*~Kw zVee%QfFGJu%qx9x!W{ z+ZiBbTuvS)VtUjn@!B~^MPeoX+Wa2w?+xNKw=KPVZ6KsmYp7P|6ODe`wbPCd`rQ0O z8R&;%xwJf+`}-}?AA-%gWFbMUgZe%d8}C-&aAAJ2?Fe&7NT8VaNCj1qT<9f25=~?? zu;E-<@6>iqktU4-4zK4T@;tSegaO+JeXVy}GqL}H7@4RM6a{=CW6yhg-dSq?&{O;D z0wq~LXv0RmM&SZ56P&`zFRD{khckDcb^a0cN7loxgQK^9^`6{O%)I%ScVlFGPqo5|)1FP$5)t@8R*WFghjvlnmn8LMgvW8LNr%5@q8R#`C%1cT@rL)TmEEYwIY(~v?KfL{9^hbM#`Mn|WB zX|cs{{dx?Qt{an`5zert)ESdlL z`T;Jt{`KSn!J1l5>}UHZ1yR~ZYJI;(;HB8N&@gMRxNr9SzNY!CSM<^+Mt+-%9bk{P|_{Iu{!VRon#Ub z1KIPR5uua(fdMWz=z{(hqTqK39DI2GJ|RGFA8{EEMHS#>#vmU*aP3Y<5PB_LDUF_m z_JS(fVru@}rO)Vm7*~!}yV>yk=+A%rxajSj9UdJF(3fkUxIzrbz4w5|lT)~vZDP^` zI!`e?a|6w*E8_b`MPV2v#K)|LO|!020y~2Ob7xzz4Lw>G(Gl3 zb9^uyg=PRNl3S1fbSkg&kP`X~PQsj^jw|fuqr)SlaaPs}Zg_Nz{)L#I!LhY6Oy);q z6Y$0)a4_@Nb=uG0gsOu=HlitTt9K?b!^S!~~ z_~2aU40yN93;WK(a@+pCDC?aDNgGDEWsAgP3;#;`4jyv>%D`sV1H>@GwQFz+!girp+r@If17j>T(d?)AMa1GHlr%r z;H(gYJQ(iSCVrSa4-2uZg@ZEaI}6<9K0>~@jOMj=)Dg!Zjy`4s<_~FjJ+fOQZ$@B_ z#vy`PIx#;kgzm&`QUOBaB8BccqP};-E!hMslT^D0zck4nr1Or%)97pm@+k)lZSjAj zY&I^jPcG%97jidDKgv zV7l^|XUa3oNl110R_nJnTAZ(wa}{7mBRz}l_$F$&(lzv8rVMs+YlINmcvaVGUfuU6 zL}Th;LgS-K_K=%6EpiZQl^7N@qbI4{(H};^WcI0dQjkmAaJLA{Z^}h8(8Ky0W@Y1j>MV0I6H|$Egj|zvcJ&E+ocg- zjWz=s<8*#g(RQ%kg!qz|#I6gmHtqmj%p+#zJ}8VKJX+I++0Yl1RDpEbEFkPlVN1h4 zD84kiG(}h?YZ-9Q#wkfpStK%Dot%*4OHef)P4-Wn4ELPN;^OxIo-SRIAUOW!CzP!Tz|y`js}k+0>KYI|ac5y3Wtq`M0AZTe0EkVOTGX2Vy5SM((j$ zcXMz_0pE&rYxohIuuS});N&zYIM!NmI3E_rFu?*soZAPj?i<^AwW2X*B;y_P5TwLy z<<%4uOXEQ+o2GVfyn$vu-8XuWp$VA5S$Mx#$wgf6RNdL&mErhO#(|LzONNQlu2Ndi zWPFR*j5h8-yjUzJ^6<&6raJi2cZDn{fsL1TqX4^?{lp*uYLuolnHX@^x^(r{#dB1Y z8&qgteJcQ)+ox0K9DD%e2Up|5NTz;z{4y`cPps-bl%fu`^RFYT+-FtaP{HQnY7eqA zTqM(^#N#c?=(+i-BB48fYA+qO6G{6an!}N^3=ddvS^1f~I>^nroKDpM0>ye#gto@R z%n#%7A^r3sc~&GOFe%mv0L2F6lgh4TYhn4{NU{HJliQF<8HOU57tjL4PRT1M$^}@ z3W5asK=1DalcitesoF7Mr9Jid)RTjE;^T;gECZE0JG%X8F=xkzthZsO?YFr+sX*g- za1%3HnctgxztoVadD!R*!9+hePK*0qE*8dgGg(`pY|QhvZoNnqtcW(P8h0Ve)GSpB z=uHh8WM8Ehh7^P(($~Qg!(}dY%u;pp4FsQ)a)qb}&lE<=Qa`FbT^nlFV!?1p1dQ1Y zGZ9gXl_TWl-0O^jhil2^5Qpml>SVYv1I{;{qjR0&FhdB-g{pQHHt(!)dOCCY;5Dbe z;`n9ruU7M5A<8YtpAe;UB6G8&XdxY)$9M|YeWr`i6bVx1f~2!^?VFfnFd|caRqE*m zl$oJhuW4K}Q;vn-yLvM~V?8(FjTIo+OuE}gq9Fgj|1$f##mM&U&fClRtRdD!Qi#dQ zt|vCSwEi-Q0{hc!%}hi;|8HzRFs z4-jZ=9)e+$+Sd(_RT*LvU=Q3^=_-GvzmT(8I3(r6rkv|xe93RJz%&ZS?D*s;Y`}Qs z6Ml zolaD;dDD408G5SPdDTT+j`S_bru$qDe_geOVt94aTKdF5=cA{fyFXab>K?q;5uxgb zlGn$CyLMZ?W#JR;qRc?t75X0|G!40CxoLieDS5h)gV*3`n?(O9+R@e_i6s^Pzv~fp~s;x*uoiQcRiKhyGJhNV8={cC9_2G4@;i z4`g84{G4OsaB9?~D@a0{8fi>MhNm`w++{G@_r1c>}wYsOnHPhq|VX%n)Ja|uFcv4Z5o%*-Y|mD*8t4zrnZK<`P37= zW3(Zd_f&5+smrw1YEHb=EhOr1LaXm58slHzW+^in{N`3MesZ`&d9_W8H=2~IUM-gu zVk9zim*HF0{v~}Ml)1{Q$ijdcNXBtXK@?s~-@qA2P)fE~y0Z#mtIVygwWB-&3I7v& zJs($Dv>5S_b)PC;*7}bWA+_5!JhVZv=7%Y9wT?radb!EEuT)atCiSA=qwOjhGZt9h zj3R>NT5;h#;Pyl}>|BzB>{5IIKVHQRG6!$vbc%z45-u4-weDJ{Ppf5)MJuB5oLiOw z^dO~{MO1DbI8-1Rl7`I|p!m5tq)hM5MJ0a-D|*GKTJ$Dcdnokum7Yt($Vnd`DdwnuFS24<^GU z+VdKtqnBt-i^Q?Ye(AC+)DJJa9maq0;{MZWEVB0D7w-Gy)m;kW{mSj0=AsujpTtsB zW5{Zyah?*SO=mGZVDt`%V3hD`6PGPE^B>CvNEqspe16?-eAkfoV(EytK+DF15#slB zJx8@ixL`OSl#+x0_pBQLU#!%A4?R`wWnaTo0jVCNK(S#gJny7ZAhAKb zJCrdEuzX`e53`tVMH1Q=hIA!b@qLA7hjfb{3DV6j*X{aA#z;`F{zax%{vUC+<71EJ zmT>VC5;5H*{kTb7l`wT1l#0balQ#~Sw8$dtwddW7ON$t98bz@?JBN&fgT+{tpjq{F z(_WnJ3sI-kg`0d9Ug~BI!)^sfw>nDN`-r8=_i@P=8+P`oXROT3mtlFs6%d9&Em!wA z3az%|hNt@jh6e=C0wqe<8`Qa#%B^s9{W?8bu!1#`qb6$&VD^(=&qJnryBksqns}6O z?odP;dNN&v*6f!!{6=kz9mo1daoWV$xDpGIf6Iz_0Zow{etnR#@y1uM&->Rok;3U0 z;qb=~#m9&Ybv}^2YyF3~_tk!8$T=#Xy&s-m`s1HpU&|j)J>#s^H2C=PYWveyZ?B(U zJlk)1g$R|SCzB`z5jJOxijnhuz@Nh%&JQ2mn8h^AC^zj>zfeQ=~cHySVEA zaJ5g&svbX)8QR3s!gLIEy6Q@}HrrY&GHtBTYJ%k)#(=eA9ubrhSl+pTOQ#DZBOv~f zr|ak_JUi6ZtP}gLJFo--^J*R2tKY`?jruVQwT>BfJ5c4yR2t;hUtNmhVXRd{IW>bi zP9Pr!?&!&6rj8HJf-*Kh_zFx7vSZo$)ung0DWS9N$9vm^{6G?%{xcC0EhT$$Y;BM& zq!uP@%RaFS2u%Ry%L1{1N&{A}II+$wKhtCIRr>CCzNnXOt z)GOH|k6%O*%vX>9X@fKYM zYZ=({J^vyDc*#&tgvLZzmiGF zm!Jmsug#)^mh{rEVC-VR6nlhBC?}fphQ8yCK3lHt*KbAIzT2nPLS*Bs>?(#ldt2QZ zNB;ToSpwJE$qx4}88bs8&nGbM;O12Q09E7go+Sicd+(lKKHE=Ad%jLz5kU6WD?|*) zU1P3Yd;XqlR?{ztzboY6m>EO*F0foRy$iPbO|7`_3$PgAe1%Kgu)BvY5EoR$)HTI;Ca7i0#V=wFwXn$#E43^5a9o7#qyPYq!4 z{1d02NaByY=$q`n>Gi%DSJQ`nas179z5ghG$_M4ro)A7tKm3PsDxZIIKwS^;hKOeQ zd7mMieu1LYa-v#;_vf$4q5con;NJ z>&MxS%Vim=arNnMx^|i0?YcA>)8O#%+kf=l;bH)9mR7&~U9b0-fBD}-eU^9V)<~3h z!B7Ls9eajT5jvLjUuc{nkelPWVKz7%W+%s_8ZU!$?R1*}2Pz$qf~QCt=*Ga{@E94& zVQ=qf@aI1^zh{oJW0n(qi@vJK@I0~SlH%8~wpczM9TI#(8sH@#&ZGAke&7ODLJB;zFC zG{N7W>tPS1iFh_K)r2v))$O*hMh5d~zFLPxig-GsDkSEVwoRn;ZPoa>o&B1qq2LC_ ziAlWdwoUGPtJTPUXAB4}5ba0r?Sp&UZC;=F)onKzX2a7#c6xr)L;o^>A$RONKEXQX zo`36uY2PVj%18&J35ed3ok`}sm(Q+n^uw_cjR|602T zU2p0;<9Wdj?{;+OQ3xT*r5Ibc{TLI`JI()p_TFVjwj;|D`xQq{nAx~DaO`c?H6oc0 zvXHDyF|%3KKx*JO!Xv_)9`2F;;O*OhfYPugnv@0__N=Q0n$;IH^(FNsT5H>JkGO8@ zmNF9%ga$;^gLDr+53_Cix%S%qN*_VT`@X-@!_>rQ&Hk~PDm@b68nCs#qu= zN=T)uFH$I8KU9rveC^N>54)l^cJehlC$7{I-aU`+mnxRMj`zUU<>^|gc%>=U;ImVY zTN5so#2aEgLTD|It5rNiR6H-_Ftm__@TTgoAJ<;Jrks->r428g_?@^%P1y2rXHyOk z-mAa&H~8BUbN#yuJ3PnymZfv;?m*Y=jh)JKp(nxYFRbQhev5^m!^XMGrm4VA`2^$Y74bpD;s;oQXtym6(`CT zrAR5X1T7$zG6o1sQM#Xk=Y)Q04vJRYJ(U10Sk;w-VcD3%4*H3pa^^%E%C zsfpXXZ$y2|rP{ht-WSUH``HOm0|bWtw0Vqt8P2 zaJC@J;pXurPWGm$~3?%MXZ=wtqcXygxoot z5^S*ercD9*QS72?mBR(4ElWu`LOi{&Lt@{3ZT?zPZF57Du0nvXmz?lApwoPaUWhg< zoOk6~v~A&G`}4K0*JUf00>v_(dAJ?fc`x21r#y{v5|WE`5Ujo<5znQx8jdd@0yBB_DiqJw%C}DUU2@9;�A`c*hJGS4(m(BB6cOg3JKi-Yf_>*qOeUc z*}|!WuE!}==N(5n1*e1vS4xA4@2b-$DfBQKhz*q8LL|{*Qc{FvDLIT57{(#ti}B^L zBsip_q{36dV&OcBmrq!j>x12$6}sm_4?t3`pLX_X;lnA<4!(K0mS1zm3S?W!b!*%a zr899D;eRN%0Ada8EK~=KW+oMu(t?$$Kxn~|JWQq%Rd|WPUGJ|ci3XQ!8ZT9GN=h5v z-Z9P?)TZ4UK2#9i6)J3Ryydtc=u)A=MTT^B~H@FbWhvDuO6?{c1 zdi+M|>=v5RXt3Wws{bix*xo1%8O}Ps2*Mr8v9+f4!!&xLW=3;{^--=(VV)F+qsNHGzv1n1&di7TSEOLY={s)GDqN)hgcZ}s0hAT)TA70*s&~>DO`M*H?=2Fjda0>pFrl| zx{X3QRj7?}oocaL!>9BzFsV2P%3$Dp!p#<;A3_}Lq6#DNOr}o7GUDu{>n-d@6cV3B zst1a(_~BG@-4R~5>jX*HnowKu`X%{`Kz%wc3cT+$T`OjL^@a^eO3k_gRU@2fp64nO z7cdBLD2KdFf7;a435EQ?cN0De>|t`(mZwWe^SG4ozYz*Tx(4Y9!X)&hNrU{1+9V@d za6`OnhoBT#RvEqFEu4C&i1+hN*CcL!hF)cjZZdF+KLxhSg11mqk{L!6h&0q z?s?qRDT#EN!?ngfyR*ru+tAr#dQ)jy7@v6kHr)kJQ&Xxg!+-PFiEa}-oP{Ca=88q8 zFjc-TMW)S_;!4dqmqc0vK}oEW3d)!tfWIKZWJX5iTD-e@ELr(=P&gTGL>bSiqlb~A zk}?PB!}g2B70ikD+TouHL4rbqP>2oAIb@YMhinzZ(CQTV7V~vxj<-OV#=N$09V1k& z2viHx|1$7m)}G}O@g7*)Nt`a%pe81%=;0#fYfsQbyA|cPwm_Hi+FW%PxKR_0M;@vm zt1|+R8PV{y%ifoLV)j~y58@7=u6Nvi;-L(RN~F~a|4ikcpw^s{MJ9j3wIY1mPl8WF zWuiM4IHW^(XEMr?SGm@N_WIOOL*QkO6-|>$*1^JMc@hH5PZ%tXNjwvODnAd|ng$8B zvoqDEDqFDp2+*%($o3JND6^ZC`KDh!QrW8POo5ZCGTS%_2uGFk81awv#aI*?5&>_! zLM=XCr`XI>VYWr(T0-Ky$0CAZ2octJT*yfAjxte-vkiSG95rVrSS+MNL_B1NOza!% zvs8{ajc{aCUCKMSiP!$QZDbBUjnCyEt5{_r-7!!}i9}PZtq#GA2`b_uVzGtwYn}-m zp*dldld@*Md2E(&IN>~IUxkYt6QVMinZjX=$_aN3$3fY+03!=8To@m`c?l{YrZ#t8 zC|mdgA%yGfriGUQ5+q#@s5|(VN_l#H8X${q5a74{#3RTNzi0@(24&m*V2JjZwV}~) zK^8V);Hs+67G|GRbr@b9JY*F_!byQ^JBh;pA%R0AqI9d{`|I@6oF~QoM+qv3A!Xf| zd21XWxw|kN<=T+<1!l!l*z3v^fr@6JO?ec>dLfd#wxwMOA<_`w0@!xg7$I@YL%Zwb z>DzqcEvH>%_?(z5RH+u15vv8|cfeJsLG5p8)WE6)5Sfh>UN8a&fuGG{=lBI32!>cn2^Eoso{w~tMNpV^NSrV|nFIwYBx<(l zI`swfBhMRULFiw+MQqZrS7sNLp>)L5y!!ZmzWn?D^77aJCVcjaN8K3Iuc_4EenemT znuGNH?k}MoeQ8n^yp!iP3EulPKk0?*^+Fc>iV5IwdS?V^GcVAE?Bc?yZqIx$nqR_J zKgIL=(z(+cOW%G>UwhrX^J;APH@m~_?tLfvzgfKIwmx6c$K&JWdRKJVAA?-a(Ae`I zpZgR4dilIOqb}hh$ro97isRQDaDM^u`oB%J{?Wt!1-$DYQN!bZzQ4@+CL;FfbXc99 zUQPKTqn`C>a3%+N`DXe|j@Prc$oby2v-b8SvG%0hc|ib_k4FCGcxS;W=4P8GE6*#y5TxOEY_qmKL>Ep$|vB~c*Uw^y1n}@#6 zAN+QG%fVTH$^*?)cI??M{4+{~jC&l4_RO^2AV#}~}@Eyu!f4%FSk z7z}$e&O9~bE3BJ!v;Pj;^TPY={sN#YgABUD!lq=)>#K?iqAVAO+-T3v5g{g`;>5^7 z12y5iHWm0+AA%FnNv)nqErq&FB}og6z=SwEVQzA4Mdc785h21?Vnrhrl_n`QUgyT< z>p+TlEBL$Cv7k2OB6+WT&5u=RtaH(JBo;u z6Z12ZD0SqFB~*z8>^TuMDw-;|k4llS(BhoPd?vzxf(q^FYL5kV3r?J)KcS;MRu_c7 z9RplAwZoACg1{SL5m}TIVE`6?x-NQ}zZ)LVn- ziJgxKSE~?}giGN!;`G2FEus!wh?LASdyPn0qAm&k`b4;jV-<#?Rj)_7l!DY<$svxP zt}D*Hdg4>NPR#Nr)Ioz2YicN}zsD0HSQc?C(;YIHDXb~UBwumW$s=1MV&)3wFm1xyqi> z!7IwB6w+H%9gAh%Chh`@Z7HiG2jA?xx*RR=bV8@^ehpem#B%WGekXhFiUWbR!H`P4G62y2$&Fz5D zWr4(RJUm_q3Utk+Kzwk-rs1vaw560~c_6cZsZw@`yWof@2qD2#UUv^$b#!2j*#uRo@yuZ&Btlq7_jj%E~z}kQKy#B0d&yXq@NDv!I+zY_NEFn9k-^ zNhJA(a9$+Zc{HO7h;T-9TCX|Y>oork3L;ooM9#IX}S7euX z4#jH|6x))-mPK}L39|#&(+{|Uw_K1&4^zl>VYW}Smg5K_hESM&*P!8LnbX&-jx<7~ zNy1Ozio1~2H?V6h3WzHfwwB>Y*}WkYe68K!W1yQFoK(SprCpA$jT#SB`ywScYApV_24VI8|IPm%*yj)7bYDKnACu&S`JHu#6Cxeo7wR~a{#VN;gwHN0!Dp4jl# z3StVoUUV@4!XSMLL{kdg(^|nX6pKhyRZ~8ugY$l^1SSib93m98fq|XB3ctjx3j8A# z?PSD3%A+8>Qqo({xTvfk&L*5QVc1e8or1*nP)23L5}iEdYOw2flC41@(DxN7Z>+mi zOaZD-(it${p}N4?tT3#*86Dx{vRAQEO%g~R;emv~ao}Cxji;bats`N;Bn~GcVuXhh!r#mmD?%4I8-oe8FQS(< z6NdnIQhYAal!oM_@NC@^cs8~#)9@%sVQ8cBf_N1o6GdV$aO;K9XXmX-?!tmG=n`Fs z5$dFg3#oLd+$TtYv;N&Ue9TAp0l_Wvn#Qdh{0Allu3J-CUX-IGOY6XH$SZSA)#Hkv}63&TrR^6nTY! zrl*PET=p9lmchkFnt2^!ZHrQ{1k9-^)!CwSIbdJ$o|3P@iM}`c#%0S23sS0m#qH_Li$X;9P@SqI2zy*)@e5uJSssKOm5C7E;TCn`2~vtakR(LxFyCQxIZPFl z7*q#CHjJT)7VzRMiz)m;cC^{zx$xpuI=U>Wq=A!SvH~hm?#gPx^p>F(Y}*Tq5|l?7 ztU{wIq6|gl!Go!(W8&Ma4ODayZcjA(D^J&CF4rjaRCit_m{_P}2&8YC!q53O=NjtY~? zeUq1`NcJ!sx~hwhm8o}`QHIbd_pZRdkl>H05yztZzGSu9{Oa?+2;yq=%dbXQUNJ=P zUXec^Uh%|49|n;DyDsL~6y<-6S-k)E&L<=^0x z{xk{n$P@kl#4O!0g!jAM@f{~>VJqIvSk_K4{wt2#UBB?h?oSWzY{CHtBJ+=bcjWvx z@`V2Q-W#HZN@M`!_zXgmg6zAImXY2Og%s=~h7Gh^oZFTX>uYdXZ3_|?!^lVdL z&ht3P9(pyO{)}+katk?)h|5A&*;B{VzM9WHQhLu+`!n>@23q}UZxfC-%O?);+S>5l z6E@gn={($;l)0Quf$M*}BU+X~CPAlloHxOmC++6PuX^&t;01Bjm1cW<4Ix-&Z8#TQ=^e~oM_I9?n0%;L_6zlknp6uF&*^6I=b&%NS2d{o9tkEIRb_2!@k27#vR8o#Ze?%^%UPewQYkMOC<*;DV39cKxKQ4=mh4`K!E1Z)-OB9h;Cc&pHIA=0*2`4Wk9|SuHB2=Mxc))sop?sJm*rHQ~XSLUcyPo$3igc6{#v)jb$5UHYls8 z(XxZHMj_b|VMrR-X-$M!l2f5rX7~Nz6q;L>f+XdF{|@b~v(@m%1-1er&$^tXfpsJ{ z6_H5JaF#`l6!DxJ^6@?QGD-MyD2?%LJ$#kBw6H;elcN*Q=1e*F+dAQrav~NM-}Oo)Pj@j6>N7?xpV?Tyx5d9O%lQ8 z*#n@`u*~zyEHkuuwT|`iCoBpop>tL<6KiEn9DWF$Ec%C{$r7dKGGbsA=f)OGPBO}G zjCh1$Vtgu92eW~ad<%jyuoU&OTq8%GVrTWiNuAAn>NGU`_5=^Bf9kd165T-aF4f?&gkBYaD!Kp;E zlAKWS7P8(X4Q3>$ae(&CsaQnQGJWdUIiPGj;^Q{ZscmF4%7Mt~H9VM(5BNKhLZn0hKdIyq7zGWD-H%Ap2jrvuiP zeG3BgPQCy|It?922IA3}VNVo+G&yr?RQbNfRJvl6<*)$JH^y9>A_cR9-h@JF6)({g zDM}@&kn+G2VF&|Q*mj*L{NX52u1T~`7Ej?)^UYPp7%l|V)_8mfaH(||31GY4Si=!o zfK^)ohK62NkihJ791DZarC`)l;=PH>SJOxkMg-M#V(2r)d<#Q?11$+)1I?>w2=NWR zskBs(LKP<>LLN9AP5C(T4$D{t*Vpp{Acc8A9Gb)K<47%LuaPKms?#W!LXu`72{?R1 zr>Xsf;7JK|B|U1BE6-D)sEHtJsu_^&Qr9F9-PqTGJidGJi41&7IN5nCwV3hMNef8= z45g8b160IySXkUdx!2x#sgdB6Nh8cgz-izXG3!jczK}yWKVg_AF!dr9>!j)wFhQ?c zM@X@u`;v0dz_SKbQDyms=xj+?@x zup3qpRI>kseLm{_So`rRdxe`%13{$)38d^%>YJ#gNp+f&RaZz4^bW_7jHjb9m{!|l z)Fl%1fklrhMaBHkq((a!u*`|cDJ+@Na)^zB^Pb&s{JVNBDGYeOsdI~G$&L&Y3^!f0 zKzi~=DM&S2-KG7YyRF=)*qTz|(ZkYJg5#WgDw(x`3ab1fLc?baD?E`!T6>@(cx4BooHM{~M@igI zAYoHewH)l?wMCvYD>=!Yaz~J+Y~jenmIO(ZTGLKn`nS@ zIUWVNs||f=-8W=5N&+$~{SKA*NEV>UoONL5n4&S~vklCsw!w`&*b+O`YLiT|?2l*z z3Yhzn$YdegX<^RbstImkh*m8_4Cs;w| z4wd&>%Q$GkOfc!GRY#Y1b-;4l16L+rgr#llagkANYD(J_4gpg!1IP7fqD@1e3BNLt zms%`{F$Tz#Wi`0Jcs(|Gss2=iK20(}lU|jB8HCnkhp(ji=e{Nt4u(k0tlIo(ew4%O zNs%ivlaWn%Dy?J`jlvC2ll8637Aa=? zJ4u8^_IO#sMlgl9S3_uGnb`O!iE%Qi;#IMa{iZ{jxw}eM^bm6IIr30B1{~O-f?SWY zRaLLC%`cIrpQYEK$$WOyCPm%?Pt*@7$L?}AiT8|5M|kC^CH9xg#vHP>ip*C_r0x=i zdLrswIS|Dr^C)OLg4jHylkDU&R#yRqQPm2!vJ|$%3nsUkreP}TY+Q2~%KF_O$m0~^ z$%wur>kubtHsx`^aj8(5%GD)x)v$e6|G~!IDj&3xwt4(}PW~*fKK;|txBovH?&lr! z$zOcCBjEOVwO{Y<-YnKfdpV*(jas$ypUw|#kG4NQJFoWZ<#D~8$$0TMtJC+p}PNI1oo%+>bSFSmixowo5kV%^6B}rpUv<`?lB>#iwn>i`)FLO z_G6N(u9oERU2h+k@7BlV_IP;nde!k(i(f4_8$8KzzkauRdA_grSntmtRu^FE>m}@f z)Th;Q%XgYLk}SjP!{cIgeBAD~AD$Nb6@kKhrguc?-Yh=9h3pxy8*=w|%OBQQnV+vV zOS~JA*w~h*<%jtd^h!PBv28hSakD!-B9>zD`R1O;+pl(;yE8O=-hktIeXtXsncyzJ zU+oW{ae3{(DFS#we)fhUjQ{2>jF|R(IR19E!CT?QKiuEr&KtbKo0lJQv)=6>kk7lr zVf}8y=RZI=?=HK&T>N&w!zRw}alhMhX>NDh!*PGQo#o`;egA&Bf4e@s#q05nX7KnA zD*|YLi-p>+-u_22DF1r-^lr7sO6+&IBj0h?NrZW9&f9uM@_u%B#{T2kch8&E`3<*Q zlF7~oqaX0qW@R|$`Hg?Gxx@Z*FK@9kcxSr|-}EE#f;dpRxQqSq5C7Nydb9YN&~V&m zd02f!5pDYb3El2USG+OY_I$rCy$~ZiOpB)v$43M8&QkvEitNh}-Ls^bo+7NgA6(Kw zZy!wdF3JK6()G#qjN-n0wZd0FUHt1m{vSUxG45AdZVo%F(|_9S=T*lGY&LjA{^ntK zy!@m;(pN^YYpCPj;Co2a5KCgXpnmJP<$QmOm>&dM4%jc zSx1jy8F7>F4svla<-y`Y8XTkU*GD;UGg%xYs_%g=cajh%PBjt+II{g$ zwAMwZ#z>K4RL)h2$-4I>Oq%1sBq3ss0-njILd2FwBCEbuiDS29E{05rl0^aQDCvVC zSIW!qkc6P2(yj0}!Y8UxmU~k%oP>H0wsWeW3BFoU!8*Y|PQb(p!AcUK4wn<*>UCt2 zZ%QF~vFcu9`dUC58GKYZe1cGOuh8bCoERo;a%)LR8k%ge<;X3SXsRUQQNyKJ1oDde z#Z7vSku&NHaX4*W=@88Hh#!(LV5@p2f{dXm@N4-Kywo{pe$3=KzE}0f*O--DIkJBa znbpY{uCsz$N6x2|6ZtYp;D(>rQ$c(N%ClYr;ec_KgJI*RMA|{gsD5L_!ZMo@YgVWu zQf?=i$0JZF55!NrGwb4#P7)#bG~moFsBYwZ71EQ7O?(oXEKaPmsN`*plU#NK@?>FH zh_a62d2y119B;~5AgUNSMZ{@Av0i(k?-Y)}ksn<`E1=ziB%LryDmftvF<;JA;sOZ% zk>xnDLJj#_%Nf}Salse>i|RG44hiT~uQN_mlBhW-eL^*qVB2U&mFbTpW>(QS47XF$ zqYCqWin}p|Y1opUsq_twyP-&>8N+kOk2E6jt(G>|}@I=}sNr2A@=~3VU$?q43R9P?JH4lGB9xZlZ3-t>v z8>AZqB}C|s?3r>E##MQFF^(i8lVRlQBtFliVilK*^JXLN>oa(Ad3>1XmQ+3NP*lQr8oLJ~p zqvFnN7E1jD+Y}TZR`!9c;fm@8MsQHO{V?PF>4>ovjTubC+yHhgKK<~p@uGGOFq=-Yfx!;4kwhr0NNOH6Zy<4mvLr2nFXd#% zFIGglAm+3Pm$=drs;*NO!^_asH5qLr|52qIRorcp1|pjJHTPwZJ7R0ae7ltss4A*r)P zIxQI?w{l$AMJflMu<)t~;7cOUkDLa|HYt&GVN1-Upc5=VHd#UD@Y})5>)NCFDn?0> z{@u%YJFwD-UnHLoCm~3#y9v%fP-#g<$8xVYDp|#wbYPOKwJ3fo8?Cl!dYWqgR8 zdYD4GdUf8c?LlD&IJ+Sig>_(hEQ^N0Z1bu<{ zsUTy^{{%xBr@^SR(+UgED(ym+2sPm&7+^*OJqWobHR%nf4Pm~re#{?9eooj7qCbWA zOQP0Ea$ZJg0ZG|x_aSBi|2?2KcM5g?jZl4(2cM?au8ZDw5m9xnXHH^(Fj8#2jtMg`(Ze( z$tfocsQe1K1?n_J5|mTu(lDUl%@^b@ieLr4jTfVH97h<-A-gyvnsUb2$sR$l z#AK3KQ8i0s@~kT6yHFLx1Z!Bwxmu>^pnWB|Kq0-pk_FbVQq&cYOfFZ2o}zQEU^I>l z$WjB76h`H0n75o}w`13TqpNy}-ZHt1GFBH11MHUveIPlMA9Wk+A6VpO?Y&v72| zCj^FMIUE~u5{D6cvx$MtA4UzCva!}(H5eVrT(U5;QBL*S7PwRcZ^hJmEIAcQ>^p=g z4c(%qP>7jRi40<5^wdN6PO^c6yt@N)g}pgq@PQ;}>a{Zbqw1g@CQ2GX+jF5@9Xj`&CW_Y3%)0b=VwB9I%z$Ai?U&h{Wc>iXXVDDb+KFgUKwT zO99cV%7n%0<-13?R9b0fgFQ;NAH0Z!6IYIFQjR9kXVXv+?1zWbXrv=goaCe^a@c@1 zsE6+8a=5K3cub6Ul~oG#d6I}?5O%n68b$Z3DhIlW#IQB=C@3FIbyKAfkBD!WS<-c| zjcAe;1P6~#eFGI7bSE~Wi8_!0n~mj!QnmaX*xDp9dF7NpAr~Z7!m}x~1e>`O)Y^GA zvI&}CvU24Sm+y=^TD^Ngz}W4%*_<3X?9&CRR5^N(6#BY523wUy z<=*BPs(}pKK^rPZ-4(W!18c<_qL-eXZ+fX1pM7< zyW5gbc1CB;fW?l6TkcoO8D_WNon{Q?)B5gibGb~|-@QWM-YkBzy?n)Ae|A_rE&tPQ z--%KG_3!`jusaae)o?+%CgzI`zK%)$BN|F$tc*3EM9<@)&JKc5k< z5BT+Ix!HVJyj$HGvU0mva{@iFrFL!OSv79mjuJgX679Zso#yt52tgtV_lx81J@I&l zT}Nr&6FK~T`QeOU{QBK$zs5a&yV|a|htp=+VZu*DdD>&I_X}bGZ+H0m6)M)ys)YD% z|9D#MKU&!@mWQ>029qannU{OFdRT7fs~uLy;|zz~&19pW5%qh2G$n;Lx)A$`Ywqw<<;LUkB>zBuAZJZGj;LuXe7plX5KEAPtT9*`O|s% zzW89zvBDqBkMy@hV&g&jTmRzed3`2MzO29nLfs$##5;80bO(MK^8Eeow7Hw{-FSiB z3vl`Tk~fPlAMpW}V{0$7G(h!!cXzt|2z~yEQ}FrE32`-Q--S{8zIg?5{bGH1+&y3L z?f-tUd<{q5tRC>j6!X<~VO+lVrx{+0C&LN6T`eBh6i_bkyVX7S^#vEbyS+UtB!l=%5XJ+3#K`4G*UcUiMHHjfB4B)`~@Dje{=Z5Pk#1h zUc5j21V1@G;q(29f?|3CPi{QCxclaN+?Thg|FO=9b+qpV!Cbrms?(>YvAx z3(qS_^7%@3Jr%sMo;8p=bVt?-8zSs6qCRztqR8T7D+u=#i@awatg>??7my>>2yK&H zl1e=WKiTo7wSg)zduGR#!Rfc`lh-|<~?<{S|fJ5)0C=%KUl(}$fm zkzgO1a0baD7bWXz;9Lw<>0vTc4n7P#SA%13@jnNGiKI6NLQ_GAauTbdL(6BE?&6#% z0nrbhBxw_Z8Z!O~TuorFx~f? z)yWYp!|3-Of-r7U(!iEol_@4%%MEq6o>*s+N=+I@I6iX0x|@XO4av*#DHMkwR@an} z>eEk|Jb-F|O5lWbIeVH|_Mr*3rz0YvUka1rI2lQG+7jS0jL@EXl>k4JsKVM55g~q3 zIr3KYJ#q%n$n45*3~nf=L>gg+U@t;kh?i7BE~|yoYB+L|$h{}arRs0U)<>>7R0`f* z2{DPMvt}{DMrI+anFFqr9w^XtlPJ(&Yo|$;&3!E--%=W_a0a_V>JgYLt8y9QAlX%fE2yJaZk21c$(CiA{4HAFjyhQ+8}1 z6Yd2rRutBX7+UFGG{|UEtWX46HB}~cta-{wcP#e?J*yy+2X=ri^^%Sw=v(X}eKDC~ z4Fp{d@SQHĠ%gh-BON&Wl@ktl9zlbBxSaCpK%TGUT3ixQ*ZZ{E0Q#vgbb-T~=l z0RXmM0(RM`roh-d3qwbW`7bnmstV37C$Uw$ewB4-$VnKJ&OKk497&nPc`KaVAuI_= zeg;0+dV`h9QTBw9qc%y@R3O8$s%ki_t{dQQw!DdRyvI{Qn6ygO!c4be*?T}nuwYf8 z!BizskhDv(Isk_!Np2B15Jl0HKvk1+N|Y9Z@v{jj3CwpZuKorOP4!+Cm^=xACv2(6 z6^ZP-7%211$t9Pd&;X51NKsNz79uH>q2iQY^(LHbAkjWp+;QMS-@-y{zChRILVgw% zJGbhUCQs3Z9!S0`b_vVMlTV70!bZL*eO*{|8ff0^Y0023&R~wU4i2cDf=|WZg=99P z)v1jNKy_Kd1qh0~-{WY=?c)G;%~79ZM_(c~XXu_a5y7GLvSX4+*#S}50~n%JWv^JX zP6>r2LaZVx5^PzKb0MwaZWwGk8+Ay7axghGS#rCRLryLuymW}7cGtXD69PG3lys&- z5)00hD}tCD(Wz6NL5>p*g<#S1#7e3+$w3#h>Cq60BEBKJbWn{H&@q+*7#Ffel2EFY z5KS8%P4#l&N^t52UNI3mLL4U9O2!5#r&O;%2&L65v6InUK}s(>S?) zu4Wh5N|mf)OkX8QB4jlUXH z6-OP9d(}^agOe=;m^`8!^DUN(-8xd+!)5hC;Y}H8N=MGn1WgS{3Uo+R8yIJd5V<>^ zUib+k=WyRFV$KyLgUkfuyt^(&&_gQp;DB|w&8l^uFk@8@>A+5PwgREw!l=--$+~-! zZhICj-jii$RgX{Oz!u@hB={J9&<5rkv?QaCoN!K6$O)ezRLMGsqcu>fbw?Besj5nu z$O|Q0@Fbi{BYCnJh(t3dFK8?=XOYQaGzf^``KW{|fr`^G&bgmb&*m=GnM}86l6W5` zBP9toExp(h(nFR3hBR47orhbP!7NmfiUyhdu1erJ9!?IgvDB7KPgenHon39oV@^Jk zBFAv}SjS25?uPCBkQ9lwVfrDcCR_$(D*Zqxm1=?uB}|e--4{4&auTRVR2{McY&k1? zsIzWkYD8@^W;m9TOqv0ilZm}*c>u+w4AY|q5Jrwtl(W+im<&^Oy1=t3QlV)PF%3jE z_PFqb;U_6D5h4l+lVn3?@~RKZs3R3nui~F!;(519wGPcE{khonE$T6`lsK}2+D|f@ z*wq(}DJIR1P+66c2@0NF*u{~P(5pZJ2;;Y&}3yB4U zyTEQNuiA1F2bd@zx#yUAMZE6m2M0B7Av<*Ox`k4%~!%gTyvHBvOnEgUR6*?QTp@LS=yErvZ$Ftb1)G%E* z#X@QTMzvQqH&1598tjY!XiPNX!&spC1(TeokU5Q14X$GJr%0QN$9qKK>m!DSZ>9QV zt&2zvF$UV#Fn9w_=rPa&Dd@NtLX~kbYW$ z#SJMw3F0M)d#<9NBU>A#FqV!3Vw5|}VK(s-h%Z#0ogjR8^WoL!e>z_G|8=z4=dUhKlFGZ?)0*U-yRC6vE>D{d zKz-9eaE82Fo{_eT_0!Y(Vc-A!-G`Se8Ln@}3U6QGTlskW12X?E?$*oA>h^e8yyu1A z?e^Q7#V>dJj&G@z6RfY{q>v0H%okfYd`*7$AQ257>WD_8+L~?w%dK* zRmqte_|^IRo81uy_~+cPFPHe>j&F{q?G(@R)Bb1t%@ek3-lWg(i52d&$NR(MrX$^t z?^mmSY&rnH!~Tsc7uPy$c2E_I&+b;(k>&pBGj1rq=;-0j32q2?h{Sb!f}4h&g-UqY zDC-J1gY9>av>#SyaPqehs58L+&q3!RgsHbs zUax}BB0!eSMv1Jkvd53pv}E|goIHGc(bU2*>CzJ*TvIU`B(jdE#1AKsO3pCHioS~4 zC!+^cemZAs4r4F#4cr6#K!-J)@g>n|WiqFKwD+mifn~^9*ClQ)Z{jNgy(Hki#b-7r zjh-PLr-6&lypUoHzru;GoNqmVpeLKA!CI=UZA4v>M$nj;C=-5LjMd2=dXDuQ!*h$< z$$uoryuvs20!|l#jrSf;LnL#c@=S9W2PgN(^-f`_2QMSp;#^KjlgnJ^U*M<~=XO57W%$<&hLG)GASz_PE5=U9jcqQ2`LGtY7ga^s0sp_qEmHFXBE zA42t>nj-t7VjXg!PI3?^xluHviBmuoKxrChklNX=;}DvjV+u}XaLGp5RH7DvYLfl$ zJ{Kr`;TphY85ksaFrki$4!LvlkkN4kog;sp9z$*@UX2MsV6UuNCF7f7y@@QusWr5A z1#xr7iislogr)Pt@V;(QS@)9Q=@Q~FBCzW#=ZkXuvXKfam<;rS>d|Se{3sZ9rMM|i zf_ZD9a~&t~$z(F=DAnXKH`17xgX)81edIiN4Q7gs-mADZd5PpKQf8RtO5>&)KMe*(6QUrht)Geul6{!Jvy;L7nE>RB+ z^n?^E^+P6{(W~S_dcrfrFACm4_U1s1N^SyH4OJA!ZpNxOh^vjomBhYqLI+>owu60m zdW^^5K$(eZc^L9)P9NOnV&`DCauy1j|m-b^08==;= z#bKs&ccycdkck+Hrc_1+3oFeK`D`Xtn+bHm6>9P{s&LD(@g+$5V}u^^-Xy)#L4GCmn{r7a2SCeq4G3aV zelFB~&JwafCwCgkYqnUb^rclrqgZ$){YU7kRFyYxbt#z_ zXdG>%Qb-0PR4N?)_s|<*5|@Z?9G!Mfb{_`1Oh%(-6Bor(rDQ@OzJa@xhS9pyJlbR! zfn}0X320W7vV>3B|ume4FjesVms z^ElXI8#%ls4rRQaxNYDs9w-5#LD5+p! zd0@pfEFfmdS)Vz3RyZioo1+BTdsLD|3k|{4tdlK1i8v9pCiE(r8ii%bM$YBH&ByKy zE<3@*3Nnxo^UK(YM^_@9tk+edE3k#x1%)s7BsP>I(`^2=crr(1fNB#69QPFzu>$o! zI~COHFczZvRKFrTYPeH(-x~H_c;#$OPeUSzhrU`4dUQS-A?6}U6Mdb=$ueD#^5D0c zULnv~g_PWs0WaPSN+vW+0;fa^ne9FapcP^p5g&nO>ZFt;=Y+vWj9#)lXF_;nFlfCK zLfs*j*G;e06J(+~iUMh-1uo8!{$!Y7V|>ODI#THsV!uToO2}!DbM)|fUL{vtBMM29 zL8Ms3LowDIyBTM7gOe%;J$srgweZg7s;LtcIi^c^G1%K_sGN0+Rd^fG`;{Zv8AbJm zFEYe2vsW9)Ek0^Hrcs8L&7LJ@Uxj$y+T>JuB%TLX)M*j<-qqb4adn+hP@NNfPJ6ja1LhqeUQE& z(dA8WS+XbsU9UvmCiN&cobI;FCXnn%78*DS5`Kg_4y5TFXr^p~6PHGUZOI@ciFzoj za@-q1nnNnmlub7$N?rzbjBa73GO}gA?GX@*qvI_S8Fxi}hX-*IDQwQO)ZGacHc1aT zA*GN=<#IZQmt<^hCyG? z4&vnGYO#&ij~x4Et2hy0D1vlt8b+R}x3d+MgzMCel4+PMDn;_yGu@{KU1bPp5>jpB zq?FC~hydD9=9xOVKn!w%7%4S|iLCrdWyw}%#1Sk)CghA0KMgf^+Y}Bq6WeQi(X#mx z!BQJ2x`%`p!=t_#lAz?;-<92Qw5LK2s&v{xVh=_uU)QNe140t|(8r`y&0j+Vq5 ze7`z&V$Gi(cX!JV#vprGd_{8D!_OA`m3`SCXX}fCnu2eoG$S8P53icE+k62Mp=sdb56Be13X3 z9ge31fsm{1-Tacj-kp9R1@Dezshthg;KO$>uevcPBT!fYE2DVjX6d`K$-D$l!eguBTBdkceyI&B5%9ZE0CUFFO zA1M4CrtBk+DEGhqUzqq{sMh~}-yWNJA zx;yexA5V9<=iB3IdCP}+wbifrdSBsk|HGCmbl~fM1X1pPh+DirUvyr)WoNMc#c98L zK4XXfmN;lXO>f}%ovE*v`~_Z+CB-*qRIGuumzzDLw1bETq;CiBn$j@T^6vSxxnP!G z?z{Nn177awnU|#)|9ZE*+im;(@C*bW*Wq#OR|Z{P?jg7LyA#d{-t3j!9iSFYw~tgZ z_?VZu-EC%|I0TA&d)mJvWwJd-U(}fZm&^7wUWQia205gmshLYG zpo-ufVUe7e!h%g2s6*Thfx4yX7Fsr;*C^5u46K99!;|Vz@h`IZzluQ4aZ2bg-qJSK zM&K$85kx`HPPMd&*o{bzLq$G_pHxLOYc#KFpcT$*3_~upS5lbXCD2p#VX%AH4a$+~NSYGnMH#LoibFGu4^yawqCQ|{GNPfxFT2v2oOi1C?Rl5iE(C6OW_DagB<00U|f4K8cL|D+(vgg1K@_Vi*Uf^6!aSZ*trZ zS$8FyAh}#+^HDJl%P zjHM}hc~0PqGGFAH4gXRmGLkz;7`D)-ypNksD7ea(R5P}s?}|A$)zIvTcaWfY4woE> zo6WQIp{VjBHa(PqKihbQdaWHN%n*?6fyc-v!DoZ zU8y#4ZxpS_d0-0SSeu%{Oe()1_=L#at{M+2iP*$RE7U%+ctLC1ly-&n@^0Ypwn)R6 zRYd-mv)J7@iYMRL;asf;EU^Cp369CbDrUpO<|M!+cv?c}_wW#vb9)G@f*GRwDs#3JJ?GkqY3< z!3#k_16ev!)S-b_Qq`>N86T(=ekMD<%vBTop$wIWSYv@@StY_fP<%}sOg$nzdUy-5 zSA*%uOf-zYHe*a452Y{zRKO%rdx>s~dMs9{g*1A)0SeYtu#S_WT7$enJE#r>PRb2K z+%)xgaTsoUc1njfS#zNi=>=8Tlu6|?{(!tO3Xxd~Ihz%8d2(T(JnU`@i6C`j=|r+i z6;QEwFfgrk+s28H60un5Q;G8ijX99ClpA>bKKOCyZOUeRs*XBOCSkJ z!hDm2qKx`wdjvfAaYTu&F+3#6R>UzjqNdM@M;vk^ygbJfFSKvYSvLK|$vEI7c9^*W zl|o>*dMN@$1Mz}aqk6D$pE5YurXbjwrM8{Z-c?KmO)6##>V`D8%n^O&&WYEQL z^cHgJxkTz0Sf!-;t;iZ$*>-^uz(ES<_^8B=%5j zO0)iRm8ijy&qGGWB##LsiSr}zZyqos-JQ+8NER(v0QeDdQZZ@Jf*NX)y$S8;=;FDO z%2RWUgv2;U@0mci6)7$R%y;OBR@ixjg0bIdlhG8}CSR>-k;{pS7#>K_j! zA!meSiZ%+OL&DG?-*ZJ1vt-I&;7pUjqVPpn)K|YCTUHqOGtIxjbBPiNjm9vB#ZCgj z!wTU@3Pafx1$&~{Hi?^)MAI;rZIVtUTk4CPEmYVzq{hi~f?|>4B_V@sMq}T6lXEF1 zsv0L}>Et#VjULoCrcDGl9aP;w>D=G3(K1RU#Ks4aBP2y)8o1_c?MqXEZE$SuRfDEs zN`p#6l9hED=5_Zq34CU?jquY{Tof6Y40(?JhKLpwP|6_N1CXbCk9ksN4iQ;W!30?x zoP;q3N3m0ktt{^;x{bmq49fVx`uZYauRhDs6v;!63zQT|9Apn7yD!P{Cv*dxBnYX> za+FDuNJLKS8C!-d{-{(M!a2vy%YYCzbBMs&U2CRv<5<-znvKT=GN0>tbi_W2T=J)~ z5QH&?p?|-7yQKG4-Y(&EA&Z$OOuI!nD2Z4zowtG2cf)M39vcwurMwPK>T9xxB(gH5 zI-sI~;2|D`F>(Mp7JDoDPtO%*vd&%6_0o?c2qqx=fWo zQKX%a+x!)U6EY=!bvLtmkT zygF5ZnYpH?`oe6fdtj;3z>w49D97o+D$5T=FDqM~o|TH$U>clPtLQi=J1I7P&xww5 z+)ZG_GEgkoH=brwz~(2yAe3)Mz`@vE(hrHzONpvmNs}5$k|=blR>aApYS=+xT@7pb z>hnJxko$-2&tG^gzdCL2h$8!s1Z{q{_}k_F%-l%;-tPAH^t|pMvtMi<2w!}AzhAGm zcbgB3FHZXh{F*qo``!L_H6sHZ`34s6R``r4TjMT0UuJhY?hRr)^WE~AB+h+LK;T!a z?f%2!JA?Q2do5Pm<7z*{J!fRw87fGOU-Z*o@m~)3Xw?}~@mX7}?gM+WXEfw~Wk^h8?7TdfJ$VOCp0>yJW$G0jEt0#l3 zetfGluPc9OysrO#VekC(yjdO34{$}*`Z1XKw7n&Wd3E>3fWGgSoA2Ll)>|GFLjw1` z`|WOfT>Q=IbWbK_LWP%S%(CH_2}S08wj>P3<3JMU=lnhXdbKBVlw7w&!x}H{eziP5 zUVn@(Pc!4~aZQ{kQQzm=8-V;C?e6^rV}4kCw>vz{K;z5dIqxIhvL7H@w|O1= zj@+*Ix2MgF8RtE=%RTpl@atELbjG-LNyjVV)W18~OWrSUk2^xKZS^UsB!=!MYzrOq z3QhkJ+Mcoi^FJTGR|?FP*TCH`m-{Eb?5ykVu;rn-A zlf;+Lc!zD@KiU4VAz=Ca%ozOwKz_Gc;`zx?eU{8K0{N@eyt(<P(o~W-$1rao}J{k+y3%#{d8Du?&pm(!sEB+cl+~rbfYH>OI2W7 zV;{C8N3wa0vY|f0u8XW@f&FxnT#SG@RTd!;$Vj1SZWBWlHGRhggs`e`nJe?1A|FOR zieccZy+?^7l-0ZqKlU>AXvmL!(0xgWei1HaCPkYv_V9lOIAjl<*}|oKAY!W@Hr86={vdiZJV>C}5egRpEN#ttH(lr|innN$jq4DxF$L z0jKx?DjO43W}nkwRK^wp<^50r>{>h;GfXb3rz##K+rgBDOF z2?Fr+tfMqZA|#^rhtf^Ci4pN?jh}cdm*le4n4tvZ48oDWp~v_RYD2A76XUYpo&$D~ z%Vr~LQ@z6pr>e=UA=8Ws$y^Adk;9xaxjDqijhwof)MRu=jiLl7u*pHvVnKkLq-o=* zZRQnlVMGEakONfiy61$}fmzO!L?x$^GWp(7 z1B)ed4hm=BVd_r8lm_6_WQaZ%;$+W_pW09Zv=j;%eI@xofv-1A=`;0Fn8guvs?G-b zs)>IT)GS<37RYt?J(N^aI@LAU>Y)HS8KD)239Gx4*KQIe9L}F4>#t;C;v7bGP!kK9 zNm7;(Z>MlcMprHMEzkDQ|I$-iu>2At!fEzNa+l`H1|#KO*f}0m4ilYkF!S#3A8o%r3WmXlv*d z>0FIoLRgK|oN8Jz#vl0$oCJ=q6FEpGM-@3^BJYdB99Pn=iCC%Hn@eUPHg-kD!j714 zXr}8g$PBk|#)$8iQA{LWT@hiBMMhOjCw7g9sx!d^FL3Kr|6YR)9`9Vp#wqgwey{TL=`jQD1UGF4UbO^I~MsSMVlOEBkVNXVX}OCak* z1jQ=(3f_N$dxbuaazz6zfTD7QtR^|!fe@`A3)QY9B6j4|+>%Cy+WszNtyG+x3>bqN zuuxo9pf5+N?O}++oZ|9jNK8$Z+heU*PJ{^LI2S+Ru@l|KWab^B@t&=!1V~8Vm4t#4 zM#~vepCS|tYAn5L9C9RDugH@rvI_7|8_$klC%9hiuvMbVyA!UV3w8zT! zV&Q0WPPW9YlV4wj#LWE32(#K~Yo(yrxg;8CIia;luI5P4N)X6M(m|^^Z{8ZMlx}8= zjGPzYRcmO!IS%R2Ph~quG_KNtzJ(N%WH2GAdcMwdg&RXzOe5uX7so_)3Ke+A#`B=` z3TfdaogxVWRa^weGEW03^km7K!7$ryap}cbg#Jg|AgKc*-B;n2keg6?iDdp9<_paJ z6IkpgJC%* z3HC%1$5ODP2!LFsYLrQ{2K%RMGJB4PRsq{6I*&mPGGaAK#U>o=JqG+;0S9X(5Js+P zmCaEpPExIN0?yE~HYf84&eJ^~2xtTdFj!}~$oQAcwnte-V(Wn#`ViP^JG7{peyVol z#cv?9>I9fYJqsXj)KpW6o4>lA0;ewbkR)uvczi61A(b3L&gK{@yjk z07*?071=qgX(Q}RDK9tB9NVUpX*hItfExZ$YziE=Ie4#@+Ndqcffr;OX+i~)1)w#F z146K=h zxrp#cLG+Vlo`tL;I-)sM<_W%!$n#mxpsYD?90pSvN>)j|Byxj+Byn=`A54l@xgE|H z9*Xw75~mTX98RDYQ{;T1Om;EZ#zOWe-2o0~U-7MD)j3Q-ykVi&CG;@&L!pby;wlQu^s zSv}-LRY3wt$p&`5IbwqvR_XA{Nrt-rfuLMjaAz}#@|~d@Ip28Hk5HRL+ly^X$xFoo zora)vaE8+jS%?wm$+=yDg+V#`%ve_xrk#{`*}wYyPe=Ive}PHsG`|3e4Y!&{PjE{qhF@BKW(DZg9o4@=@*u_E?>)#DA$l(yU5Vzb*m%&f(Nn{-KR zB-iI~yIyUHd|q!~V5|2V+;n|-ByN_BsC|W>e*EXV_3AAz@#BBPThAcP&u^D^tEZWD zb-g7EB$n&ee`Q3cOo5kDLuV${+<8gWWeZQXpWl3oI z>TgMM%WG}=L-Xzz9^?K7`_<}i=|F;)ZYk#z3Km*I3r?R@=KD|K;f;e5;{Z*SE*rdT;WLhbM&e z>+KKA%^F|Ma^>o=Q?t5u!Hcd}2Y%I|o)3G-2e#3uW;-6WFCO0bxQG{4TT1gpiGta^ ze1w1q^3zY+`NbD6a{lGx^6B~T_Q!wNd^r60&#NmQeCzUdX|W%t_)C-LQ` z^E(q*%=5MRVCd?*^R_?laO$7d57^P8Jq1zdwkSK>(f#Vd*py$u*nO>EVD5IO&bLi; z{uVF(yxARQO!YHfad+6^|J}ZvrsMja|9N}F!6rWc`z`U_^R+(jO6nKe4Z8?dokV)^ zbG+AnceHE#{IkzD1jORhKcC*s5{;j(iRr)naJ$((+d=Hx2K#;PRN&>m*=>&xyUqO? zFZadXUh=TmEKfLuAK|W_mRp{MdwU9AX}Ngh{c)3bI88gg-ge>LfB&z4LkK#ah9VOF zJm4Jln8|3L32} z0!S&o&oHSA8 z=0K{dNP82<*{ge0!lhJ1P2zwwFixLy?NLtC$5;(sN5DJzV*0GV!u=omW@k!n3qMX&it8D1FrG(sx(fpfley7)yA^>)Ms!rb15uz zz`<1}gcaif4&Lw0s(?tJks)%Degg?sJt^HrfvBlqC6$#Xm`S0wk~M9}i3y2Q;nm?K z?O_OtgN%4iH!!GlHAuF8ITzksKxG+NO&9RVYAj%~nw-d-JT4`vfr8LaP%2K%%3w2B zQhX3_t$8I;h)gY}fl7!QPhVDuN-jLr+Uut2Q|3F?X`sZXJitx{Iiq=EGoq%#$bk?$ zRn+6zl?8^7h>5$tO}&95SxS+RxBy4f3(dvccO6*lrq1SGG+c{B3Ihn%Ja1=s!Qm$~ zyK&G^<`8hguEfC5@Jlu1Nu4aJ9;uPhOu=cLX596GHeA896Q@?F2(5U;Pl~9}V^2nE z!O5uJA&n}UWti&*-_-6B+0Z7d$y^s1`$q-y}>~ON?fZLgUZ4TOz;|$pe7DWNr z;#X-jJV|C{+ajTI>k)u~y3J%=B<|sUxG#WYr*Y zaEtBr9DF&n)%Cryaa;lu&=R?*6=;er6-x3sr=p4EArvmROR1tp#&8)m0$JrXj}_tZ)St zU$`Td*vl9eBA2BAVG-E_qMGnB;ZbsoJf{OFoFH)$Ee`DNwk2M78b;1jl922Yh#XZu z5h<)B45k|5Q{2RQi&ezNurNpfMIvQ)75NM6pYWd;c}k`~1*wH&+lCxs7fDwKEOobm z%wUQGTwu4x;LPq48C8jR8B)XsMWrDT(lj~IA(CcRxmY+3lGiiBzf@-^!JcF_%(%v^ z5&t1toGYgsbKsM-bfhG__9YtIKuD@4eO|8okg1rgkWL(o&~~*kK_y;2k$zS*PT-Wr z0g*6SzX2Os)FL*H;}-rtxhn)nW3uKWG1-VjeFFok#Vu1CI*j^DRD|$Km1D8iynsGdBDw-g5g>v*O zY^ZF~ioH!t=@N3g=vrcB7@#23%={(EUF>rl22}R20l|hBo(0iZIMM1r$EZfPFw=n; zPh~f2l3E6Sag*#dOg6~HVhkrO0$EqorSO>>9A?8UTJ}_>c?i>%%g!KDP(9MD!b%T` z=zLSi895Hou(N|mP7P!vX$H-XaCmT_6pYqqOV5a%W#S^q9G7GNjx2qUqjn$`wy&5k ztD@nG6$k>@RkkuSU#QAwW#T9e*;)m~EZrn{T;z7w+*f%ZvFRR6_5%r$-2{=8Do85zfJ5R3sLcbjl)nuB2y3 zGPMb-I!Ue#_G1q0o86%zNr9-Vape6Fk@R0s&8lr8Zb0xx5bkYxqwz2l6D=k4bxe<& z_EOL&q$OKnr-=DI3%vcG(mncy$2MtE!2$R(_ zhQX3sn_&J)Xj9f0>VUF0CC|*v_<18Ml~P!Vty!|=27WrxIODxJbV%3X2<1=|T9M!g zMG;nsB#}`PaQ;fPhjKyX}?|DC;Y3;=|7Wp_hGj)v?<=gFMe}--1T=r{Pmi6cXR0I;={~( z`m#dL%l-29@q~~6)9LAXrYJq`NGtm8!{S%V{dToqZ$BpgJ)EANI;SWWtDSH3Zgsny z;jr_Y9*)b0%k5@Z@3*TTcAFECruNV-{`dd-*Qd<~Vzi%j?-qyE`oGLOxZEFBBsHBO zz87pO{`B^br}d19e#PY3ox?U{e}%iYt{iHLFRW@m}rLHgF4H;bp$@r;cA z>c@XMt~QIi)#5we*!FF;S>2M*cDeXw`C+&3sNSU!Bk#DXe@#*O-G|K&e)zAvjrMZoiH(Bb`QgC*!&(h0WPM+&I5#ZMtl&x^aye*EVnDS=N-`E%v$P_HIv_2=*%i)t z^PF|1m?lWZ>r_c1*qlvK?s5zbqi5Kyd^-> z96;I6v~$zI`)h9!AoFU{S0fvmf+&ZUhvwUU77Arp$b5n@$ieImp;Q}2dhsp=;e;2Y zI>wwTS)hyMX4JfBtZv7& zq)4DzN2$;@iQ6R`TD-(Zvc9Wg(M&LrIz8FS9*SN(#Yy+C7s8~4ImD`mfgE!#H7RI# z;*4XJqpKWxqY_Uf3#wir;Zn^KKAZ}tqpOvzjUD>pEkYaYZGQ~R zBxPKxk)^ruGzZQy6t;zWOl7J{N|U0-tdNo_anpeNx!Lr=x!mWSL{Kj$?o%wAx+)aeeioMo#= zVwplF{3CQe=iaN+MhKjwKn`Vuo}6!=1+La=&`G!yg@ZMVu<4F-YzOLXvlC9ZX6C@x zM9*d1UiL{Sig*`ejfystlSInU&SLM8OOtyANofP^K(E1sPzUz(t~EIOR_R4g=1c|r z%y?wM^3Q3eaOIS0s?Az1#-Bv84yp04cr10OQtJ?~@du=$ll;<-$VXA5NCEDb=(6G9 zWUqL7*p-(Ds;OSR4Ni@bB`>3#$I@yYCdN%21{>6zTt|3a6B-#0JC%V&zZY%DxFg7e z90RiS0ID@FFpEy6szPUi&GgWkD1Hm|ZuU{(! zNsbjr{xNK?eUqqbl&V1>hVVtQM%o~C)#60Pk|98(u3>=A&IAAR3~7o33Klw;SX|0;2{i@qoy60z8{ZtcTftw3Kcy-?3L%jN*V$P0rjo; zyeZU)$AclRQO#%!6ofnMIP$`)>G9Pi@=1xi)AbdKzzQnL(PDiFn=ZNg$o;7xmt^9} zs)ov~n&d=jQe(N%V`U#)l)P(R4Zp6OWu$!Z;30{_*wOB#lWAAZ$jO|rEjqPe!8k`s zY16<)8p*ETQW!Y5XFZGPMUkL4BM?=BLdMl$7Vu=9aBXNG;Hv~8ic}AZCwZK#Bzl9e zNbnGjw5+;Do*$u{xmQ^vq6%$^^rNAFt9|K}n$pl-Na1uXhYzUKC z^2pqs@Mq&E$X*i;wj#R=Y}$76sR(9OaG1z}1Wl-};C&m^K))0d!oqLt)2G!J846p* zRz!*fRhZcw1n9-A+J1Z^NwWgh zm@hk^b-R?ttl-lK%Cp4uS0>r69SA-yCtMAQgvmGto9rw~>=~4eU<3w0Z;g-Z`q&p0 zArfjX=>V*kT+U&O%Jb*QVpamgmB{p}j9BIXMV+`}{#AWKpzhpun zcV`@01}B?3*1Tv>QK(MoPaNcR0O&N%TN1YxQiTnS`cmUAv3 zK2a)PnBqFv1U^?yHt$mG6`@XQn4V`$IZQt3ufpNlqPv9xEIBqBDNt1{hmC|mappj4 z6#2aT#t)sBQzuK;LqT+3g3{)wyHGy#KfxP1EK`72g7%K%cHIhUn zTd-}d$|_}}woyF*k*&jC+0aaC)U3K>IQ44Gy>zw04VE+eI5XkX03MHk+@o-d3lo26k3sk2V zNLc(5IWZ2z@qS=(pLM!(1Fc1Of!Pc!+#&KGsq-En*GZ+zEcYUkVx$$;XBPpE#IzRc zHxeKNPR7#%gd>sg$}k!VUojrzQ;3n~RjxsnuB-z3rKf&Q`VPGbg~x@**e%r>f)GENbcy@!o~ zU)S5)&FSv^-k zh(Ydb$1_Xp4huz+*w6RJ!-Zt{)ix3ryX^bO4`!z1PDuL4(-J%OYMlvt+^=TbXXhzi z@44=8U*y4vUk`s}S0haIm8ckk8}1T8jZaUjr(MUMV&^Y^{3ZVMX?vz_J#*YHHp>fW z_h+xstTVom5Nv#V>Urmmd~n_I*Ld?w(g2 z)XFC<#F{^>w_C2)%%Qx()*JKe)ADd2!L(7tzA)x?#P{xpwJ{^lWX`t+BPN!X(+>M% zy}djZfu|H}LRjB#xgN{;SneS>+rAtKZuR^_KT&WClq(4S>EI}A$BAp=rv2;rc;R1}{(PEmZ131_ z4)a?|V26 zwz{_E$~GD0rUE!?yk_qn@Q2%rfO~y7%=g8f&3}2A%FoI=vO!qeG+o>EWUDso`!#Jq zhbr%0e80CL7TNbd*_3}~;a~G_|M5SIY4-Ey=>01*pY!ee9rN_7m;3w2pGU9TdHn-r z$j!xRfKOF~et&*i=4LIcdwN&1>Ymr(Z>N*=a^OtB#@0(am>YL6BI zt)(I9#ORPwyvCz=f*1v1M<_kk*-LiFFW^KBSg^@bRDuO52Ai&$_)-E8Lm;g$^Mq4{ zr)i3;&4~3tQKY7xk_&?{=>#*f%`nvdt2?$96r&E_7)e&TANsZ$tdgP_k+M;;Y8Dd` z2Zj0O%rdFwVL+88< zKsc5efzE?g(iF3gDF&+lfD*)R$&8j3W+I&g0EHz=Pb=La%rh;%{6sb z@PW-OYDcTU1QZ! zQBu1Na<#OsLPpT5q|5~|#pFSwNN!DzB87pcic+wms;MBMRI-((S0aH!+l52Le^{~& z?TAY&2#V1Suz6K^PgT~_SC%?0NGM8~W596^W=<>3;pGPz0qI?kK6HI08Q%)3R1SPI zTG<6ZzdkmG0pEA;e?qo5r12WCGbDus?JQ9z#FR>xSSev5Ha zrzhZ}#Mc8G+`OxX!1h^C4diCg>DysbY97=%Nz9ja`bQue>3SEuuD6Sw03*>?TL>pn zQF3omVkwr+Y(hi<6_NB*RH%~qBj&&@yxMmtesqUP6eR6jDM zrnI#v;?F}QA(Kw09c#QRqk@r6l+ri#Ex2xH0EmP4Spngknl_=CU2wQ+#g)^mYbD}d zZ>iSKe{!fL=Tn(de;EHCM4M?JmI>4y@+)F&5(){(6(D_~9*b((SW`(3gOg(gJoUXyI z{^qs`I|5GHE>wC8k$TSFLYBGqh(49cPYP=l-SRIM!U(Wh6BvgyxbU zmG}xM)M#tqUvZLiQxWI+;hL+~7Pmjr^a%t5YUN_lX&kfY?m($dN@^G%2-{p~#pTP< zj_$FhkzGuGKU7xkMR9}%Vi2|y*Ww&n^Pav$pGr_4h<~zl!w8RTr=ieVs9PLbjDSRs z*Ak95i1;1Qtg=EQu`#6N0W3UjFVz_&_O<1cP}LwO8+=+op|iU)VV6abq=+Ya!&m@J zK(oJZk&a908)#i#90m z<`&hr8L6q&{DtXLvn8>srd^_q>VaaOwrK0xLI_AiD09IWk-(^OvD8?+O1z>&O_JX@ z8I2hS=ojckQHM-$>XU}JG&=t@C7-fTs}!(?HjFj3ylyT$C-W^2s&}DREVc<+@KSI= zNh*Q%dm={6ozE6Lu!K@Zz?KOv6|3XIH&q)Ed8Lx2h%e!RRSrRJ1yKXEP!~wz8luLb zFo-dPcAR+rTBOt%AyrVk(gsucO0laSq_$gfQz)q_{jQeii=~3olQ3p*Q3$BYMoEQI z2{5Z-iK{`q1Z1v*^kK*b#pK*NwINRy+4N8fN}b4MFic3&ho^uisIR&wbpF^m4j%?y z^w1QWNB06LHdO_k&MHMUmBsc4RtPat)mx_(xuy~_N)>~#zlajY6}v-mkW#Il)wt9f zcghLt4h`ZqQMH~-@$wu=3U^ks6lC`X0NMLH^y<%;GC@&{{sJ<$;_?{JD9vNS{F(l;q z5tRoOf^E;J3!Dy$C?PrE%n$CvVwbQW`Na32(i|#e2$w))I=#|@S^J)+9YPY{G(9wef<8H(ryIsZMZ-4#Wv^@!StB1Z^y>vXWSS#gvV3v=lQbgiqqd2^=66c6w5?QVJ5Cwk$lS+~BM_J_O8qdo7# z!+t&--VffwV>{W~=SgtACp3*MIznGi$BAtA_F0X?epBcyu?@abAV- zc;`UKlVHaS5PNqJ?$DoZ`qNHhVB>NoDHZ5&cUU`@=zr@MGFNR}{fF(^M&KYe;-}9q z7AD+R^CU3x8LIqt_0{^-t9i5fM~%dV&ykOefY8(a^%IHjyxpz~+c8r3{_+WrY%xWH zPq4b3-kc}l37PzIGylbYX`}6ERsH?zy)GsC+4Zt|zMqfl(y>q~Rr}r1X>;3ApcmL^ry_r}4yxUBl`-OjS3Sj^2OPl!3d{})k zoo?oh*j88NxlWtakMrLCvK9a8RHpT}_r5snHT7fBOYe_%jooQ~B{=K1)7JlM=l}JG zdAnQhKVOzWYy@7n%fw(WROWv@AN0`|j~dQUM&VO9{aeQj+kDw;o{p<{p;ibpQnrW6 z%6pGsJ|oc2Ys?;UeK;NpZh3OttAF9`Zoj#C2C@BqvfaP>etkV14m-WZ&H8p@yL(;U z|2Ok?zqWTg3Jz<_=H}TnulKfw@5Lj0d?L}lTJMi{4(-_F!`ZD{R%TejcZL))l1Fx@u8X6M_`0lKzbcayXA9Ux=-h#kz*1)`E{KS@y z_ykG$Orm6zOfio|{fm?^8YOj1!G^*wAU})Bq-UfY?nP@tp^o|*G3Pf(`xLXS$>M5C z{3oS52@;_OMM^0IEi6oBOBfp^AcP|dRl?JQVbN}o?wS~yA!G_dV?m8X@Rvk7LcP*E z&_pKk8w({Vtd7D>!`9IlHaO*~QImw5Eq4)o6XBh$>P#7kEZh~%hNc#Y@P@uc%3rB# zwpbwVgQ%V{v8Q~a#B=MBANzJB;&0bT&6GM#>d=R3PbWw-#Z6|)6n8_+MtsVxItq!r z!z@XrQnNC=Y9WT$ukXbnj+#8GL7^IWHxkS_J5dFMk*YBbmEK&*P#+DnKD>$=^DZ4<3@Z-{Zt;_whDC1&qek++oz?=kz@sh0%R>Qz<+$y5RTFc+Vt z>_cg9dkIN3QsF`=cN*G3I|nW$$WDd-D+wtVl){vzlGt@kSCyop%FC8ij=)k>Q_vzo zJ*i$qs>q?^U;@I^s1b?!23oTL4LH26Y;2E3b7(HLnSJIuMghV!2vURUl6HQzNWvkS zRa$X<*SaoI{R?|}+WoczZAAcBS_$K}V(s(QE0zX;UN*I|flrjU?BFtk?Mjk0i)scW z#zikPBjZh@?F0_5T&tGHJlm<(5fsgAv~7ND4G>O&SVQIZiI)$d%yxzmL6YPiDzI;P zvZN$FYz1-4Af1IdKSXCSbmAze&j?lBB4W5D>Q&G6=a`^L7&|%|zQ| zBNCjdJJjM~3^cM1PYfeFo&S`5Ze@XJfJbqm!ExkhMinaH(VE5~2{7>%A`DpL7if|g zI_(=6#rAy|03i&84}ObIhyu?70TnBYL$H)e2!zuI>9Npmx07c?b$BTc&~7k0r-mThK~B|JFW~4fT5#LUG0nAlW^sc= z(+)<+7b76DgbPBVaB<@hl1U{|L=tp0$a{(SyP9fvsRT=?01|nU$BR`*mmdorT44U7 z9Z6MP`@kZ3iY?4B97bAkL1geK*(|OBWCV8-0t|i?YvoF`*?*&c;wl1NVqAM0Qr%VX zOIkltw&)Q7I*NIl?5SXK10L5=G*%QCo9UuSQP`MJ7)2nb$mu4*_mGBO0}=CQ5F-qBRCSEpR!D$gBefDE z+nG7+T~%iAVQ-yF7Ji{Hjb>dmwrXagnUVq%CWPB$`yJa#8jlp=!8{v{j;rk+$nW-| z6hJ`+F~;_k(6b;$@GrnzD;R~wdNJuI%BpKA#;Ow6xUxFZRR;-iRB-F^;(4$IOoldD z*cG7ApiIU{t5li*%(0`egqnj}$4yZjJP4Lglm=01k91&lDzQN0S!>2`4Ac;Awlq}| zY<{ggilYQPv@CKMDqFN_4{C^bqZ>ZOT(2Bmg@!Kq0XY`{QIt*wz2j)7e$_qW=A1n!}#WD-QAU?QdgheaHQwj_a<$c|*vjI*1F#Sp3MZ9#INs`!yu zky<n_DOCS7cpD5*UJd-KY*xk_M1uDsZI- z0i3bZNjC@M(W_`g5CW+rVhgxzW?QMRmK@rs-UNaj^nzr<6iP8{B#MOO9`eT%q9)8J zF%as)RE8^ab~x)pvT7C`MDq2M%njm?g-TTAruHHf%utV=G$N@+l10fxTn}C@fRsiu zdrp>Plq-*ZO7H8pzJ(eHE1c-EcGZptxD^pfv`CVbg`V#S#o1frT(wfTH>4gu+_^S| zwT^7@PtCJHQ3yfwNxLLSsFIo%WCRAvyxI9KQ8XC(LG&AtxQOaGMEzAYH+PU6AD!(B z0Mm9}snwyr$KZ4p@U&Sy;+~!s} zE+i7ks;850qh27wes^-ADkhOKS7Q9f^Uc>AYm ze+FJ2cK7pfeLo+Zq4h>wj`QZ#nV0m{e&?Ky+xeHPzG;Lo3r=FY;NGUc*nd83rblPW zJls#4jkDz5t#5@TUEhnFcCt$x3gGa1wcl-Kr}*4Y+k-xJYFxjj^ySyCaKHQL=4Wha zd515i<8kvuGyCnd7aH;(pQCT5&0*(g-2!6$=lS7bw(BfxuXpoiQ$F~5-rrB#m8e5s z*z?@ZtJ~AI&}Zt>kMkrlPDkiETu{UI1hxG}WV74tyPxd81t^{m=P|vu=U#751^;_x zPrtgKudgS^-@e@op=V#cKONTBLJk+$-~Sy#@i`0VSN66x2EUo_?cd%`j~A=o-r8k< zuxanRkH_j^7VObR z;?*n+?felw+Gj|BNez*ZoepO_*^KQSg?%`loIkk?&h>Hq=Kb(2pxo=7&D(mR@ZH(f z_p6tV@z%e;e6?R++dp56c33b|;evOLKi;1)vzvLkS#DPVoX@^|hF|_>xBBvKZR4}O zyIB46-R`vdYP*@%evfb7{j__qHM{!Z-A^}@&B46Az1!`Varymr71HW=f?x~yw_twD zf^#VKy-?lGzscG<&!0gJ7arH~{|c>-W6vIE}Sd@PNaQ`rI_v5!57E`s#FJ<4=Lm#v0I1= z?Exy;_Jf+XV7Sz>g|bk#ipH)IZ7b2A$lx#|JG97grj+x5RBCqb*b}--o_nI!aB*mf zu?V3h%4}2PzbN6V6t-?J*)7aCB-Ky{ESYr;GA+m` ztQztzsDGh?4>?Vlm{BNCk_9Hi$W1k_N?}E}*H5T)tu*~u0U(Oa>07LP+Ng{pHK7EH z4o6(wX75Lm%@$&8WPUnXZUMyLH{xF>IF2wT$nQBi2|1uFnyflCEY{%SyHwN}#bH_p z<^pZyRE=<16l$&jBiUQ@TXkuiPr-_e+NZZq*oJH7F(#z`^v#GGs5Vy46$IO^i>{#q zxeqZkK`Km@QuL&RFRnawJO9YSNHNY4n+E@=Ev{YLlb#-@^&^xVqWNG%rLV~{W)kxLFkQ}reJ z&CQJ%PzF!pQ+=djuu5~8xFXb<(;~ve{IQlGt-(xSQ)SyqeN~NLSn!Uittc!?^}>8e z>Jw4P7nxzQI8eX{HiLQTZ3PT==TbtXM39%n^h<3xIfoJ4s~}Q;5=HE~PDP#iN>TkW z#7X0HzVJ-qq$gIG-eo5dQKd=}6&Ph25~W3hI)*9$qZ^G*y-b7+NJ76skEN~N0U_UJ zd_$_}{Cq76L;{2g4$LLYuDMAgN#)v9m;(hnT5xweUEvl^CrUm~PEL?r!zY79QynsT zN)~T5YM>?O>`+Lqq0@rA1ohewk(LOU zh6*5GVgqr~A_GJ#k`l~%i;fq`_06D+#LeIm@kH8cp}G*g;3KJ1twI9%?OkLL`$%G_ zI#TqC_DC&?o0thoq}MGNwYe|Y&I3T{-2i#)fuqgT8E1zYn608iyKd`N27N;!f_8Fev2&5H4gd?9oF2khw*yJbSqt`uxr z>U5KDS`oYz7s7mz4z*MV!NQ>oEpZ;^pKXK4$q1b*4ox^RbQ)uWcu-J@DQcaH0oIq@ z!-f};Esa_K7%Mz)@E_T_B!e!Ju1sfEFCPX$!-7&1q}I82>1=Sae>{}Ey&GyE>hf(+ zY)m*p0wzSqT*3)pPLi?%w1$M=N3L!4NnuW3T6Xjz@@zWXb@nC}j1nNlVvM08hHQ#3 z!+?iC&aOtxemli#W^9!qM{4=hKye}n%Y`Xlv*u>m?C5)+rVrTKhkA0|)8z(43pm5G@0imr{eI&e`k+ zrJ(zq1Bh?;m#vyFP?x}ojk}6?-6uqzCL*wuNFolME_saw#eT+&j#aA!M1^ZCCwcA~ zA~O)3H*{?@E@^}yeLwrj{@dD8dJV8nqw+fy*mSrY1`FZop{gXnEF+&a@{|csH!R9)b%MnqO^M)d8OvcwwFFA8DcwO-j5h8=u7oLkfGwB_NHt^Y zUYwKEWodDV*==a_R$`1Ocm=A(AiY%q5s`eY+Ul@SS2I^Gx#_@k$)nOObo1B{qo!JN zu$TOUk2&Hu*HDiG|$k>SI3jDJylrK!KPW+M5utH)t#FJzr)yAsh zX?b9IVaR{uoW=F@wly}=8)iP&^`F(^}YDpYf zdDTd~^X7CYH$nAa+XfBzhA`>S4vy-cTyc&;I=P zV|M>50jGUEZ=DzS%no`pufCp6n~hMDg8JT_ZVQBPzTYhzt=IeA+nd#mlXTiu?U(lt zht<3Pw0S%@M)tUSJMC`{tM7Kl{k-~>@OV!In!1wnX-+EF!M zkE5W3(z&g~` zFMP4~K-22_w0||f74G`m(_yo|cUbE3jH}Py-t8_5pi=*_zMn4&PVx+ewF~{ryghtg zaG`ni`|IP**2(hh_L8SA{s zg;<_*Q^3t{*1L`PhXn=I;QEg>=Et|fuNFGjC%EbRTkQ6ZE?sb+_D(iBc7xQ|bL}2% zmHFG1(Ja_*Te|BH2mkl`+iAN#l;@rH_qNDZhxvN{?q84d>a*X!{NeM}^|arw?McNK zJK2ZA_BzM&?*2+p%lAyn$2(hhw|DxY#>^kr{vp0!W=?3V2C?&q`64~Tly}$Hr-$`I zH~hnHf3@3g%S%7lB~K5lD?Q4!Epfl8zV(eg`F8d1Pt*Q5@AZAdz&W57Ehv8eqj^xT7X~Q);0<2 z*YoA-htq?jY@blj&q<}TAVV*bh^)k7ScEycWeW7YL%T%W5JVLAajgN`Fos)LY ztNna9ZH|t0F2uO|d3ycAM&-Q)5{CMsz13n$+0oO}09_sIg)U;p`JAAfnb z-cQ%hX_pH@vyaa#EB0u%o=L}t&a%h-=3c#;Q^ z$&-g9Vft86KYAQ$yCOqjJ*&qWGKVf{N-a=v7{odSNDkpkpoFYyw8a&a=1w|qQAlHG zE47uTkVSezyFTuFrR!PQ2-|=p#Em`Y(ncRD7q=7!$4a2YGiv5cP>-U#q83AJ6 zVA|r}0P0`Tb84&j!F`!F*1>m)oQ9##w5-IKIP}1vX%Vh+kT~UIC9LZ~N~`QX%&Y(a zr2o>29ywEXAu3bNFsrUs0eAm4NgbHtxlzJV6y96>P-%EH4lrwb$|c%flA;4PekE~z zLKv1pKUNo8cA!O|P*)!E&|sxWTAMiO7Aw#IIUsIP2~>*@DETPP zA)+Yir9DG@QoV&CB;hDRyGppIDU68r|1^6d1)zLN$`%wONMHTbD3a^yRzxT+IvaS4{FY|z50*ND_|2BgCg*GH9` z=`W>ek^*MmMiqMj3DP+S6x}_j(E%k@+NK>SjC7PXaRS8FxgF}2)vCoE2{JoL@6==1 zxgsJ;MB3uTwtwX=GNERaD66n@%;`C2SGaIaqDClICwE>`l-H z3BP`VlO>Z%uGptW%+9D@(NWL1C`E^a+RsWM67H%~X%?jxlGyIZTAR^AS$f;&4Jw(c zlE8xQ0enk;DzQ6K%Ci;QQALkbMm`1X50ezIomJ%LsE@xw1-K6>O({B})jQSG8WIN| zYXk>7Cr9LxRd$sUqJ@4032}2KDix*cM$~zhJ7tytxsz9z<3$+~QV0EMHiLb(Y9XOU(EbRF8+ zB5-kiYXw;q%yDs(yGnPp%!fHnRWC%TGY%bgCUv%gppVQ}iQFteB3qPdaTA@nq5!QB zbuSjQB9Y~?r*+BPyL=QQVD*vR4r@P@!~#M{E4NPKDj-C zpQw9-G<#wv{vl+{A4l+41^=j|Hp`ov;cLKq5TLN?Dl7q&+)T<(goKB**orK~rEP1g zgdpyMR3LcFELk9R8ub!qZW)#rR5f~*U_^%w>WFl<7l^)50;|evklccFK&cZ^3LJr` zU9y_d2#srQP=~N)4PsCIx5xQ&cC57-X<>uVi`W>P)k$9eqSpgg4Vq11 z$idW<*?7hB=xB-|nu5n@wmkO;s!Q^a$g#+NjzS@@7ndw{gqaIy#u@LT0~y6s)WAaC zp4uH*XDXD}#k4k{G>U^|+o>;tQrueVp9h~ye2fq(4EBO(gIcAdv5VHTJ6KK-5*a0Q39)F7x5adu1V%vK#FS>=9X4@FQ30*ptJZV4PRHi42zt8ZE8Zd}z3qbIv(!M?Rl$)QOk)K)J;Rx{@gb@bUT-Lv%Zktb~&w8n11qz$n z!FD@@FhNbPNIPMkm1nE#$fNC3m9{K z#fM_Lffe1T;{!P*+fOJS3VRcP?L-~m0{D}+?&N!LDcAczYckl{Li5+zxm%Ys+&cI) z;)ZTTHv&hs?U6(%S=%`+3s1DFXn+%H?zDs}$>SLMnh@7l>1anUy}Wo#m9K{rPP$k< zb$g3KB!lBNik^kex6U?h6;e0OBA~TGG6OCLgudYb@+9&<$BNY5=R;>$)rwJ07ai@e z)m8kpAcu?0W8_>^>%|_V0v_zqh`BN-R3m$>o6zh^hB9+kV^(0voTc4Rp;5~Lp4&Eb%sB20uiiJ`yKPrHVg@#03q&x&PoJrH(kz><3j}p2(gUBGk7^tI&h-L&)PD&it z`(accChP}uH=R@k?w%zv1bJq)v{O7hCTH$PKeUy;8|}GK#|i7p_GoZIR*Zr9ILSLv z-BStx)Swa+_+Y|m&;-%mbFffS{#zBRRU^{pqq+=_KgA$gmc|wgBDR;Gyk)Lwr zZuRiQDUEU->wOw@{^g3bpz*|5m*|72{1=j)s4uLA8nrpvW>T?;(o1Ov1^-6L;ZeR$ zRKXDbC`F1Ip~R)fH4au(KDrm{K=#PQhMXilPMN zS2r=1=_NakR1#){rSw)$Ru;cG7~x)d8R|uhs+bZ@{U8`Z&fT-W|NY?J|39MEgzWpr z-Dz*vU2o?X-|n`@hiSi^?4iqh94B$#9#>bV^~Mpd3&_tQ%l5=wwwKxM=f?ujwS)a{ zf4yIQ)(_*~{`%_pxpM_yIJQ|=I6i?#!`P1t4l~zkX+{Nj3JFTt< zI$W+^F8?@fSNmzbc{@F>wl*fa{p&K|fB5PTtIz)M?x({%omOA{7vlGR);rTf{^TJNtnXYObXRJm-q z+5+YDi75ctZ+C|$Wc7pn^V?~^($&@rJoeSX-~8%$xBA2UaJmu~^xJ8BEbqUWr#JJ$ z!Ycsz{cd}@KZ8hp z->#3xdB4{J6m745{rd;I!L<5fIvfRHUZz49=|tE&4&&!#BDeEf+iJ(B_4viwhQ)UG zyMNVmY#00BF!=>Y`^_GaGS782@f+xo{x4U{h3o-h5&k4?*G;uKt->^yv@ z)t^t(di8}p`{waO!2Ue#ZF|OGt44eBw0~S6_OR!cdD)4Mct20}mWR#mt^M3_>1BJ_ z-d#P~1K*wjx1st1ssL(WmxT z^Jd!a-u=(D1wHJp%>u3MpWKH2Je04UFx*cY+5#*6@RROR1xjvgbk)5GD3DG~zQ`%# zUDpV;8^qHGA&a`UsDw6%i=@I-6z9B`a7_*JZAQO6xXr0gL?#yH?IE5oj}0a~&41{7 zDh1jtB$7=c0F}amJ3`WO!N$eZsm&jvPKlc2h_dryd_~}@TBtjEk<<(#XNKn;om&>} z5d%JNmevlHjNa)yds}YHE7*6sWUXXe1)>6efFsA!+Rf ziI&!&GCHYoB#RKM2x^IL`b5~V;P?!rCJxG$(RLF^HdWcW7bPtgSv6Q-FhI|WI|K;# zqC|%fC)%inPn;KR3VlT-R-zm8K(XC>kXjDu`$SQpqUN5O2dhk$M|L*37>bJBZ6I+* zi&HQDpb&T|Yk@=^jk?_lY!cL-N#;=+Xv0W4ETgTX3Ld>A;>WSVLzJqQ5TKO#^X*7; z+}q2m6b$bo)ObYzT{4`iXHe^YCx9ES3`uei6>!M28lt=f{FQoj0ldf85{52brVPy@u^W~l~Z zCyrvf91)hQ$r`|drsQy1s*nUTxguMx2TX(#)lbsp&@0Iqzj{%>O3(rufHGUP_MX^2 zW!wA2TXex(Vl1(9LckuR8yebAv_RQnX_?H!e22WyrUkTmMR$bWa$%v5544_;TL8QtIDrGl^qmRn+px;D)(aR z9_r+)^i<6!gp<&gV~4F=X%urZ3GIo&AYc%tlVGe^n0!UpE~yZtrNnCP0X9d*zA7a8 ztY;Bj!#ODW!8Q{ST<3fQ5QOz+ZwXr|J}k=bCS|s%QO=yN+EXp{r%{3#fE6(p#7ff#C9Pd9ZMNLK6UF|99b2~_C2-Zj~&Kk5_xdxNcn z4gp=}Vts-a)_mB8356uP!MYoEB%{J^aK(rU%At2xM;Q#GHc3L7VtYx^i&1gL0!OsK zt=uvKkq5y#d#bMX65;``K#d02!{el9vBFKb7u$1_M>h_A1o7K0HbC;^Kj@qnw;n~? zx{Pn>%$OnANn)`>Lb~T9;u-p&RWB5Ujau{_`jDd_8^~ysTKVJPTX&r(ENQW-BHIAC zM_CNywDZsy#!KXFW5ttG#>*^lPyVa)dD_ZPFAkhAXyA}EzEmp^QF#}gGUUXV_gr18 zI0a%KY)C9w2N|kPtK};r8J!)ysimG}_38{2kyqmpC*lIlwGv_v)nSwPydj%lOF|sO z&^cluxC&}u!sPWrj3}PNo|?tY3vcY8#m}_5gmgqkJu$6tJ5$B>2)o;Z zaAdra%2{=`_@eBZO1R!_ew!mBy9HBdtGl>^peh4(7z?XQa6H?n=9__ct#Sd)yG+ZN zP?1~0I!(>dSSCxs6JpGyeL#%s5_qABY+JF00(qCKeq&3ZGOktscvBdVGoV0|Y7F@9 z^0H9SL*onR^u|InKq9Moq=M3v&|yHALHQW!?$vHbW$R82P?*EW$tkSS;ct7&-?fAC z8+2w2$_ZenoET3LPNKRJh1Y5vs*;3dg>=~Xpe}dpjIxc<_ROG?%y1Ipb?!q~k(ju5 zH3HwMdy2WOf)F|Vs#Uwhuq`)8w=Nm@@uZKJ zwgGHCQ>w{5-jHC^jH9~SXpstJIKt>gbzn9`ziVgPfVgJ$@gte8BCUw9J*Ai{HPB_! zk!ak)MUjAz&oQcUM~aL+2GO=})H#A_uF!YLTEEeb73=)&j{(Va%Pn4qMKo;DjCgBJ4t(b+6)7M+%_oqoERN z6sDHa-R%&RiXhk2>~2#jgD$RIBA01swC3P(6{TVyy`l?jhET8d5CMB9c9XhjRc5_y zDz1P3K%jIQcrlC+!;#uw5{k%rvXxqB_5Bu6!;Lzzz`g97Y~~WqkR-?@7N$BiB4qMF z?2oBZhA4lxZ3VSbJyZn|RVhKu{nVt;3UNsi1oqbAWWhF!F;z`W9wNj(1nk?6g6R{$ zTLib{GAS{l?tAbPJuQ!&s!H^brHKlYdg7;0A2yXxB^OkcKFXDpOrn9$Llx$irDY2U zn~LphX;GcAYhA$vse;X75-EvrAGxVQy2ZDxyyp_?bt3lv*b0r8T@t}=W+YN(u7ady z3(_=@))AcM~)Ro=! z)vNXO#Y$wf0@Lb~0$}{{-A|kK)h{l@nfiD*&i4yX>)sKWSL^G?>y1b_>t~-fhn=px zJ=h)ZohkTGctg(^N5N}_Fr7BL1zh^g{OY*+V%okwf8~dHy`2~8((S4MVITIp`+4gO z!Qxb%&RE#*pZ+dpRePY@LRP$)%4IH=NB5VUPPeDRoowUctftC)z6U0jF?)~l{BC`{ z+t@(=V7J@-%cEWNv~}>H?)5g?W6tvW^MCSn04RdMyLHeU97g#{qi{9&NhYO z+I6t)vxmDVH=RC$R`!?q41ZjZy>>wjjeQ(ezniwFY5%D4>6hgc782wyKVWG+)6Sm3 zs?Ygjr-z4q0XPfTxe=4Fc$PmNyamCm!4s5RNL=%r|Mu6{_p8s$zW(j6_v_D{Bl&dP z-TNcn>s?p&O=p!B7JO@?e|}wOSGK?G?|05wyq~o=Y{?u8qIenR<=fMGF9zK5CR&^? zL_B@t$m`W-|M>NfpF0%44E}WV?Q}S<%Y$4msONwC@BWvzvyI`yS-|7V#u1oaOXjo? zCST3ZAkf8dGYV3`DQvVlb8GHN}Ryo&)fC(U~_59YFhg)J07R& z*Y*zA``w{nd&}6Cm36ram+!%@XMn8iMcE2x!d>mwyWLy8(ZN>I&9rw2|C#-H-tR>q zeP7V_+ts0v2kYyvY>;nu_x?-g?e<0Uow$faL%nS5gY4SlJnb)@E_OUW`^D4C*r@#W zhu?m4@kFNGKCwcV8~@2U&V64U?`%9Qs#u2b=U{JDqp7G;N~CA5S;esQqFGugYdw|z zQe}z(gDz#h@`zN$<`*@j z=&h#d5UZ4vuTrfb)TbRaBLEy#{*gHRR8J5Zi(b9!n`-?=?J9G;X;*bwa^IwzGPvX* zN<{Ux-B57EqI_b_R9UW4nRMS%+By*1xEv3ZQj)F1G1C%lrIAYMv0xz&7GHu0D-tPW z=hzTUYba_I&=;V%m58oYi-xssH?1Ju`g(NjHZpJnQ+lac;1l_l+>QvUmQ5tG!6+s- zVy|+LxjFWPRx~TxS4PG!*0y!zaLzooh_8%86gumppLCbKwm&Xi3jrI#t!D`mHw^Mh zV?~@;qBF%>lo3#6o>JG+^~9j5wUdI%HyV=;2YBd)G>|~5g=L{jggMQ2NLD+rSa%D6 zKhB1JDDMo$#= zVtkJj%64MrClwhxi3BgCNy%b#fFmJ`FNkdNAd)*Q!o}E7OL9tP6suOBQqhx@L-sof z5YUwLjQy9)Eot?3jZX;K@KF^=;HQmhB1X1FHS1Pr)5dX_(k3K=#FR#K3`Z%DBE!R= zG6!X3C1OtqhE5`qU{J)ul_=3aNTvb3EW2?fb>yQcwkM=r7HqS|R5nsKtO(Z2!{(ZU z-v>9V10IRrZKCXyB&scE-RP13Qu@No@@o@?^eIVc7=q_SbMBj}LEcAF>1bk@>l0tf zwut07u>ukX@sh&CYZj5bTdK6kXcMJoMGOq#|u5i6Yiq%s7y=vsvZX z0ohD2m#rssi0H0|o~BA<@noA+fQWSxxddE8>MQD+B{3@ho>03gzz{p6)SDCeTB#dt z^Mb?U%NV@~DU*|stty1zQHj{KC$WRh^+pH-YhGK6u!2tVsHESKL#^LYssN&ShvJV? zwFnD}!Nq{VUzGr8Revc4SZ_#k*o*p@>g!TfBSf?)W7>-I!kxV!lqMkvY%%nRAQ1RC zA)T2R{4+M^$WIxO)jP>_5dDq~r9@+pLbj=NKz)cqpDQ4bFJC7pr%IMe2O~vq?!Z(* zkDCq)(W*|5ayH1^fD_sYdrgQW2|*Aw&ZsILE z-Jps}g==dHpp@FEI)~t3jW{YsL?-k`I>kt|tXKw8GD-DF${C2I39LtUHb+=O0)l1I z%?`GNM&!E+G7&w~rHV+EoJ|j4F<1;XSeYXIPHns3qS_%Nr4In23DZJq+r69)B;;2~ z4jfmpWLI*;zTcucUoh%Zyn?K0W}D5RQH*Zx{J=V$67;}o_b}FOQod9dvxTV?+4f@l zMV_^&IxYMl#vxRBk-`noRWPoy9NNB0AvHxN>Jwbq=@a=jwMX^X&1#`XAAzz!SjH~1 zK$kEE)KB7mXhpW=>mjs5#R=`U#=Lnv8KN; z&3(%d;?co~$d0e{H_@rd4JWV923H_nN7Ic!!Lk7nnFgT%2MU_ZA9yY?H z@nX{v1_EyiyUo^1O?44Dt1TuslSChM`U&NG?M*_}Fj;&Lab_S+$(#>Z!a)sDk`IB~ z+lJ^UIc^q2EaT+&slgMnR-*BM>QR($hct;9^4aCq3Z-fwvD%kA>M!a<%tk00NqQvY ziwmjt>wNV{jhVnTF*X=9h)9|oF(o`JYUFadk7}4i{QEeJ9Ui@Q{K$(znnE3ofa)B? zjDe!s$xi-?JI3#CTk7Mf3YT_@n{8l`Hsu$)2;J#o%|Rgni;4O!!trPOGm0|^*2AcV zaysS25K~o5^(L~~5J?6RTx0*Pb|*8N_f3sFYzh@(UsZ6Hldp?(siay0l}mznQm8Fc zc{~Za)V3)|P>(ZwEi;0vftDpVyp#&o*TSV7QJXWU<9LJ^OPzkU;fRh_0y{+fs?%*G z1JM?0B3i-1kXLKoUqzGzb%uO7ufO{bivgQ-(SfE*j?3t_Qb!j~xe zWWm{A1N|M|FsPvk!ZlIm^hna>5YnciBq;uT+Y(*4Y-3wRU&X1kpqrfV z^5R76kd#~{VmLVp*u&YYN{HkGA{v5-IEYs_!`>0e%}C>JZe~Olesy3_PA=I_2ggDr zA3YWPr4N8wVlEYp+ktBG3e!S}&;I=P18x7f{rz?owD)nUEi+@|LXke)qGtTQLmRD6x!d_)x6ogUB$os^>6?C zmkXZl>v?y(pC0ZW^)KJ-3J6sHydd@ttBw9xP<-3nUc|uLLUKE87S7ND0xqAvxSF?4 zhdb@B&u_P1ivaftEW10*f`T57V%yxDgsMHh-Px@+)AnXTSHAlnHuLI-X}vwZ_-1!{ zh63G8`&+x1e_lYiU#^euemaA2g$XX)yqg={L72=ZC~1LB`qTVEsM*u;PNS-?f3>~c ztPgiSvfFvH`s|x&_3b1m?&lhV>B_#mift>X=;||DUjmPQzPb^{_PC#?`$E^d+~#TQ z4iEO3Xiskh&9!@7?d+ac)Ase@gTc96{oej_+H5Y~|8_qMO8Z7Xd}aT1@qP$)Z~fJF zHahxZ+d12D2wh=2wugN)uMYFH+RXOz^W)rU5zc-a$2!|=*zfPwo6R!jf~L*|ZCVCX z+_Ssw@!rO8KMR>^4xHaA2{UWATGr54f6+9anS5=eHjnxjL8$#TzYvS9P1YB?(`IM8 zz~AKj#~(!mZDW2ozxuUvE^B4)-q@qw&(qo{oNXL^`+LHWj5zUUX)@1~pb%6g*tji_foBr0CU>=(Aa925Vi-F~Ho@s}rX^V3@j zX1YG^_99yrVX=Ast6NbG+in!o?fvdXJ4Oq$OyL;?t^3DDxb5rLZ`X(U#o=MSU(m_Z ze!sJAB-U;R6n}Pq+ODtH4>lfGyZ!ETdw2N!(stjjEOx)C20%}ihw}h6PXl!Eo{ZY< zgh+b#PRDFdihhM0^GJ|OX2CsejX=oRJ$yqSfoU@(eu5mMx07Q#?Q z1+fN^8l$q2km>0|Qqpw~h^`A(E>gLPEpliPFH?TL78cCbQVK2;UHo zCdQjomF#tsMlxntgB@yi0{4NVrlVS7$v!j;;Bpl!G=G7`6K> zvP=y%lD9(Efk=gr5_TKv1kQP^bbov?SkwdH^Q4Ft*|#Ln+Et=b9*{{Hg`!>cp=ChY zr~)ghZmd6>fhG=H)g9W8LnWW}(aX&Bn}l8oOt46{JxI&Z41*ayWN9-S)>T~KWh1C6 zj$G0)STIwySuX3e>J{*;OgD^_KQz(rEW{1pMJ9Ysm zeN+n*91mhqv9V&$t+rN$tng3le>zk$RyQG3@r946QdqvWno-C*%D(8up;!=F>Hk*H zYV#75r{e|{VM^>wKvZrTIuu}0OqNj@CUyPD*qt%_z_JfmxN5r1VK7;mC}c^iE~&Zu5b`$TkL6XF?Nxi zSeq#Uqw1UIlrmx%?B)T9Vg!_HYdz5RQc$9f?Z6;v1x)4--mHZ*WwHWbgLCUuacpz4 zortgpu`3{hjjf2f3hAHp{m>HxNQK!tf!cT};}iN&P6t8NeX(H^@T_aq0!7Y{*$pc0 zh;Rzv=4Rq`hQk;-5D<2v>#S1(Gui2iB@BU-D{SZz518Zj@h9OjHN7y1SRNGpTCQ|p zOL0da93r+Evmk6t+&VHbooWTxaFm3zt@KU(Og1;PN4mmJkOQRRi9+v#FBUT^IEycj zDzkQqk5nj8XoH<1wsm?W&{XSJOD`^IK5R=(S)Ceq>Z%w+zJ)o>=H)}aRZ`c~xM&uB zBvTW_!@$gbwr3lx33TEBK@L0HREXIZ2dk9jV4aB5B3J2x#-#@a5`U2|RMBG=YnFt? zMx)gRv3A3|NJN4#jhb6T@%F7&Fw})bDV-8ZA`D`_2eTyvGwOz$XDXH;;`nx{3hGa? zbqDVu^DBBJ1idx6mh)YrBBcR%yv81hv(qO)8m(03Bl$db_6+C;+ODoz6Ur?l%#172 zIZJ3w2ocbOu?dPTE2`ur_UT&G!ssNZPR^S;!Rh2}9H&JEH#N*gWEb2#+~aFKCl&;g=?=rXdORC9Np$S zlG?=}2BGTJix+Hrh)(4;VwbI%*-(jSmN^u*fC2*@aCVqNLDRer?3}gr+p^w(2tJWF z-0mO&C-CPx!JdiGE<33lH4&OW9qMQ)lxiPRkIE6H1cibOROudtXrv|I+Oo=o=n!tM z!=#u*un--S6eOdhRUCVJNy;W=9sLB4F;B^6nA#rgylk;sCdi~Aaxh4+1-*ihV~=m@lBlq;gaGRnD&RSn#IC_V#v(eP0b zSb(JwIVQ||8!DBAWw}^DOjZ7-gTBvl(hMQQFr3;JsDE%t5QSA7|uAkO%vEX$n zp#*qU+lWIT>PLZ$s!ph;r2cwJdzt|@&kBc^48wXaIC!+rX8h z;1Cyd1m;Rmw7znBiffZej4!B170O4_wKgZ14hHR6!i7o%F=LB^c}~jo(1OnrXGA6x zX|B-LP3kZw9u^C(b4BG?@+JbA(y3_AL~H{RXGp~7U`D_;A>rBWY1K7Nr;_rQlml;N z|7m=6yi=)0^4tw^*$A&%6*A1Y?;C8^5YAhvgE6?vZ~d+`eLTRdY`KB!5jZREjF zU%!P{V^3~MCn99x0vANBMZYmx$Z6&QdF$;&_;_XckuPTi45Q+Rp3`fi1tv&p5Il8F znI4fdASFmA0RU<(Q4KO-zo~iowTe@@zl|4$I7uZm;7Lic*u=S_o&dC&HLSNJHq8-9 z6ck7?vZ^Ea9mWn-G((ac2PU+M53&qmpHjuq;-{Y7oNPclDmTubG+w_ak4_nN*e9c; zfvC&NjviZEYi}x5AHpKk_ew)NW2Fetg)fgzhHDo2%kYB^L z;(+vZYOaO`(MJ7=mFB{7N@qG1QWnhvM2}{e+7_Ey%#`(!Y?Eq$V?!~b@=iHnu1bLl z?NDw%;RYDy_4 zm_@#6mmAI)&31hDt@DF@H*JpshidY|8O}K*c!fXgPV;sd!v!h*9>^^8t01M<^R!)G zKLMjfT)Ute|J}R)^WkCMzQ;^&ro-Wh7uS~5b+XBwHPgbaF2A$y z&8t`QbhJ4Xnsv9ayVyJI&pf%W_Vb@l!rtyDM{wV)gm}HyTO3}m&+mG_-uMc9JMHgR zhtvJNUEp%{z%$g-#^Uz<(B&B?X0D+U)jFc&-b&hSi9eTE*MUG zTz|~>5bCd&ZEQqFYsSFY zant!r7UC`6+HL(I{58ucY3eqQzMAYOf}IPGyb5McvwGOrJHU0U^rW1jyJw2e+owc;nrV};a`?uduUnclDH7eX>W z#=x43Nj+?sG>LX#V~dxxt>0b0JL{-WmYKANhg!IjT_u4pz(rN9D3plWcXZYsv~Dj#tRrtqDF)AG0np@FT4aR`NHeq%!| zUJ~$ToNnq+$VYl;iABBy_J<<2eB4BE2d_8`2vU$yBGr#x0(>pJpEO-U3&Bb-OG090 zQhKj-T7BCNYB8mXlH@QV#g9_@Md*asiZj7U1nZ3z+oHdSEfiYq$yNDaxwoAZ+WsD^EUO#p2Em6(jsJ+D90RX~K>$i56IS5ZR5vN-iio8lr6_tt;>?oC7oiB{ zD^eUsJSbWRUQ+o>hPNtLy5*GuJ=GDT1ZAtHiV~y>Pmb-JjZ%b&xV3Rk)Zrq`SFid8 zn0Q+*$h2Zn?HIAhXhBS{$eJvKG-5%yM*cx97U#EbZEVQ;r`&a%yKDt4fJ4d7s#>(! zFV%IFC3IcMsuEN3hax&AjAMWjhJTO(3TsM}I#AE^(1{6`pl)gep{ccdQ=PQ4G4IL5 z2XR`pB#GR4ix0`7nTIe$Dm`ZJ-J;BYqdslOZS}Sls@zr?65H>UigY>fJHcY$C#oY5 zj=oo;IdMvjP@IBAU^?KwkK#InJ$)CY7{ePOMO6;w2sRA zLD``fTM!^`qMTfC6moXiz4}6fK-&ZZv-M8>g(~?rRf$AzZGJs6RH&tbSPMm>ruKVC zT@Km2ZmO-ST!^V;F(0G2r4BMY^9LbC04yPMD^)%?2TDd{6qI4AJ zdU+_X9uZvOn8QFGjoRZPMkuZLg-QEdq$FaS1&4evBLK%lC$eG!eCi~|rVS}iL`oN{ z?I@W)03op`hFM+1_CKkssZ*z|Bw+mPOk+k6pDk*O!N{vH2U{#{2xS=BTGzOERnnk= zwuWBF1gggNOv@``8w8U{AVE?H@QE$NL|U2Zy&0=O(_f^^!oCB}Otc-~ zO~h4CvuNikqaMv?6h#qHr-mdT3H-#H*Px^>x~@-<=1QX;6_JuQ7mTolR%45+OD*Y@ znzw&o-zOZo*KQW4xj;(Vh+%NG|jEKGtvvDT`~xW zL@nbd<M207^@hgnxdH2SFxs#d&)i1n#t5quTD_sdw?8n*LEXDx}BI8p>0mR z&;YXJRjCGy-tNm`jUGjbbj>3{Nns z-vN#sW@pn)jfbn(s zfvJ(@Fy{*6V6R*t8-v9iIZr~*#D~iUz|2IxFh*U=p;c7VMJ=p%)p_q&f$i#RH^H+=%+pn;cV;Y zHSce%c7YE`Q0F_u9m*M`mQdfpJHUuJQvv~SfT}C5Lxu$NePdNkS`K}ShoD9w_!;SL zniuaukA+pG7hNR)2hP+$U!71heC;Fo}{Hrho5Tw z5SAoi<`tsf@E;8HrT790)tN5 zSd?T+DnJIva4GpC0Bt?Mecj-w(N$*=ozOygCzPPGtX@z1>-lo^t21g**DbiY z0&iRXxVdqd+wHt{uGPl{CMae3f3XK!AdRBP{SZ&N-<|AI?E=$l^Ang*7caLEB=@k~ z=+CFM;G?_yd%;F^$B#jZhj}9@`&A@~6h_6*#dw%d~v!p_(0hxPG^-?SH$ z`Ch2p?lS)Euj}ms*PQ<%?Bu+?etHz4SYO$2+Up#S`?bCAY+rjbZ$&10=VYb3WekMn ze8!IZdMg~GAdx<)_v_4SGN$E!U*6oWw|e_0_T(=`wR$UD?^X=1`qQt)A-lP^WwO#}E$HpdZhLEQbvW+! zj~9*{eK`7%6rk&sjrFTDT=k6V*7KHY3k$lovCwW2)_b;jJlOr5IQWTT__Z%?AE=#e z9lN-`w!WDQ;I@!BFMO^R>_5NDUg+za)qDEc>2%y}clVEHg55${Z6CZ8S?$&4v5f4i z^H^X+np^w z+i=(GX=77zvE1hTP-c&wn7-43%bj05`}B9GtvG#Sv#bv90Qm5w?f>5Y;Q8E@kKgQ$ zPiwvWvwkY1@Od(q0f?xwLoLZH9daV*l4?AWznQnxBbrlK(p2GBigOZbv~1(Wj-k@E_Irq|0ZEY* z>jI7-s6{-GkX0uYEdoKBqguIlrHmHM=S5M1ki4b=b>()Dv>b$6t?QYSbD{(wQsvp7 zsb{vAhzt1jy&BX((N-C!p<(TA3vyF~0!^JNI*<|`#M_9RDpFC1sCLoizQ&ul=Q}!$ zTy+x*=~yURXt+ds^o;xE(Fm?N=mN~1!E4eQtWL?shKcuA_o`U6O{+- z^?FKG%feSwyRv_Jzk!rqMJdZD&zx0I=U8^M(ITTMx!{VAzHzC_*dqymGy-f^s1tt7 zp}|y`jbmmm+SUb8@tH*pN2g*E6(knO2Pu?O0?9exkuy4uKAVtl^d41cb5KFDsfe^a zHkZ&;CcM4PKD~UfvFs{R6+f{gMg%J|c*_zMaK|AbDV@|Bu}bSQ?=A9&*dY+H7?K3` zj9`Pu(Q_7%KN_6>0WQ_cV*uQkn#hNbane3oDT#a^Zql|9%c1haw20`Gu>ghenT0jK z-VV|-#HyUL7lN7Xh0LuAe`RHt)kqe~d^fO;sUKRZbemlkc?un`t#Bulj~7-Mtz*vS zOcTi0j$NB`&|DJ+zafRBm~Aa$H9#x-V>{FoMrw&okuxNOFEX@rLj}h!X0XY9g}p8A zQ|hpOA;ng5bsHt!i6vG_^$ivSlE7KizBeeJNOdk7#7K+{=J-pT915#K(_$M#Y-+9( zKaHbkW$Q$0gffP!P06{+Y}YWlBdBm>0cK zrk*jg8D5~FyVyQNHMpd5DOKjYYqTRl;v|z&mFC<7w)NOz^PMzYTEYR1y>P}9+4U}L$!>3k{jD}#B!l|MAl4E8di$L<%8HVv97?;#0xD>xawFc zX@3c~*N$Q`0Rr`G~|Ff{V(>EMuFlmb2^M|4JM11-W8WyR11h|=|?ImZ?~+h zkmg0KWgIAJybzf@X0@Tx?OWHeL7S2)=w=D4T~8J{Ptnm8Xe^T2M6UybpzuES3=gy8 zjgOw3N0G#m-4eJ4&J5*MYdwUrS-Lh1kOJ@gz+$3o6s~sO_=bdJqSFr}veGHH3GF35 zQleJ1N6JVfr{J|h92Hlrsn5}J!kJC2SVbBK^GYoq8>ZE87D;1Rg6TCsuJ6>QR^W5{ zArK=09FvlsRieMWv_mn3(AAr(zqD;Spfcd7iVU?-dS42WW(rs2gbvEP#tPuRV2y|5v(*l=rwGk+u9|O=3oI9!d@iER%Mj^ zlObE6NJ@heyCyrEz1ts|-p!t(I+XJ>5)@Y4DZ)g8S0H^Ga`B`3HNnpwg*YN)yWAJd zFr{7;yu@HFRXgM(u|u(^HgdI`+U`MSRkQy1i7V>0m``cOS{Ai&LGg~Ml*b`s`=RE& zBjO?31zN#GH5rAahE%sbQ`c7qR`Emosz~J~+uj}4gV3WrD&8v?qe)L%z7SH|VHk*} z#7=ePUkRg`vg`&NZbI5(DrI1Yy%!Gvv8S516RX0u|3Xd(rFWw3T5k6-cPdh7vsDW0^(E`|+_hTxJCjP5SBuJ+8cO zh&#-@ccCI}DPEo}UX-E?7M6>SwLb0#8!T+bA6pAPs6-+LeJB`q5(lc|I-wx35mOyS zfw*6SJ#5O5D7S5$0=r$rBF;JSxf)~Xpqn`aTs?h4uRP< z>((J{wun`N)Y=e-e+)2rYD9+W%(ACwapd}}r1KzpfMj?6>^&1YznXCR{ zDn;}XWu;V*m#8U+oh9!_VTJS|k$5oG8d?59?tqG;?DOd4b@iG{+WrVk$f-s_mBYvu ze3i)@E|WkG0}3u?Pw-N!lerXCoxnXj3yVYl?9YEc!uRh5i#e<3huAfJf7)zz*OPE@ z(`tKioTbP>=jkZE-iLYXaK+QZZtJ+ohuy=1ZoMcxxuJQ2z6n1095N@$(&LNe7rMcR zz_|-y<=XVY&m9COsOHnJDm2^~#&;5^d3C+*)sZ|(h#Z+9;W zsQB=BIL`O)0lLRWzl89-+ijuYTuoPp-R5)@D(_Fby?~_)V{JhvmJ9vcfBeq^c239g zwsw<;&1pYvo_+l=U+)B2eEPnyocH!v*A9lZJFf1g&8y|R8h0B%r!9UyTxA^g^Zm5; z2lSsj&6M!Z&vC%dheEK-onv2@S9H{&Zu|7q-G6hQGl3s3ma#oP1O6_!<%5`&?{+r} z_wKwsI8^(1Hx;DudH5cDNF8dfvCs@mo43v$yWZO4z8}-wvlqW`{PJq;e|lWq>{i>| zu{Bh&nwlx!N|@#Q8;Tp-r7v+KkhV&g+2EP zLwa71=Oy~t^%?s4ix02&G2pYBVuF93QK6P`d3M{S>e@CeJ8?dK;PeVO+!?T7nW;9^?Ba}@qY%|%h-Rsr->XmT7ej#7c)B2{Y!Sk9cPr381i_llA&DPCvy*tT~FEN(S(HPSr&O4X&?+O^(WFRH(8gPuMVJNtbPA z3E8Zsf9{jbHDQ(S56Py2b1PXY*1Zwe8m5Ihi}J>b=+Z;EUd^&7Lf4H0cC!my6)1er zQ&l<;p*`_wj+P*xU;6m&$(*|wM=%@W;#08FsZu8Lxd(U7#x$HC-qsbVTd zI-Z*$cDg~Tb;MdHhyVno(t^y@#wm+OxH$?|B~OHXxHd^GR$0|BXzLfbRi%>%GnINN z`5`c;lX0NHsZ*~4)j)%5vyzOVYEEFXL~4_XJ8kqX)lbxH(yJ8+THSh}nMO^w+`8!fW@ zV-!O!Y9$J+3UfF%T(0iaSzFz`>_amth#^vNUx_vq*TII268TXqe|Sa@sujzXQteBF zSu|2S78OOMLR}1jDjB57JIZTANNTGbF=qu;Riic03KL7UdUbx87g$Zbl+}hxZmu~~ z%@Mi5Ai;RpCuSRMSf~$2^gsv$jA4ib>kz=G0=fJ7X11V0b~O%^3U$Lrshy2>yc0wv zNahrk!3CPj$PNgncIzv82_FlRq)4(o5d+@wX)nv=>`k& z2~^ODEER~XTM06v>6PXE*g@0O_G7J(xzz6ZNPP-wB^{`^6lxK8k2=iiebPZm0A&`y zx@VZ*=nB(F&}*m8NK(a3m_J+&M?4x_E8<&5@^MO{e|@vK#6@kGVqL=gVH?P6*3$Q8C(# z>aHRTe`n`yqRQ|fk#fpEm>m~0G^|mxO>}G9=p!=TD!_1pJK>jjKA1<)mAnT0U94#a$d&25B?aS%^pC5+t^ zwdGKuD2v}AsdDO1v5-ArE1p>>!AIaHqxs*o=VCjywbSZhil_1GkUU}cVcRaqztr8iduVpcK8q5`zU-sl4{7FV{;cdq#^M3xtniL1HD%cz?#i|RyO zJEC@b5@3@4YJsu`fSt* zM{q**n?>rRw2&Q@@nR=j4hGqZH8ae5(wtP}$R+C&g&BktgK89X_}Iwz*u^QW%UHuQ z&kt4+mNtcr2a&C!=MXv?&a0tLWGts;7%J^4Uyt4d4yne$TnLijRPi)IHdj3x$UG{F z5plwh1!@}$L>wC049ICt$dt_XMi9edFhqe4dG zX*KGkvGpq@2O3JTbD};->&<1bh{WE5*k!qov>C)BfZUwDxMk_A)V4xSnxr1VdQ^g{ z|8!Ulo)d;(Ukx1*3v-gZ%-AIyBvOQJmKG=l>dEn1X-))}P`(76%|3TsC2ml>^H7uN zHw$MJ7#r0KvEVXxYT88dHc^8R^}mHHaxEx7*sJ9V=FmL7zHYhlPdY&@u}QdLw%w#W z4&;~HuFiQChrR9F3c~B7>*7%LbVy~Hm>6vFnjNA-kIX43M=ZQB+P)M!!a04=90G*tH@tZ)l8LmC@b{?MWC#9 z0Oa(MbOqHOU9|~c-V+u`F{mOr6<3v(rlWF{meqe5-R~c_zum5WH*HVT-U(L=sBZO1 z$e6Q|9@h5{o5v6Td_CRIqKDnB=D(~D$GI?-7R2!OV-%hJ!|B?oU|%_G?#0fzVTG%^ znvZYixj+oRTtJ%zo2Q!z47i=Ay%45{)75Ezbtre)UT;n}^P&FPh48h91+jR4f7-sM zTRqRin>lTWsNkx% z+}usS0JpjCCt$uwKS`heh?=`c$QRX>7d@cKJ7l<-=^(?AGyLcO=gVw6gz>m4q5rS{ z{PSa6eUbEE$A9~;|9sqh>D04P-i%FzgI(9Jpqb7_d}EQ&>d~H2e|VbrPuuww$Lel= z|7z|vaLc-|$G#J}>iYRnHYZy)uP9LC3vT?%QNEwzX)A;C?Zf_PW4@5yE*RD4iLDo~ z%hnmWS$+MXA}+PA-mH(CTSv*xTE3O6dAofcMVl^rq^IrL5zAMrhkZpoYj&Ov`%}ei zUT!t--fa&1T?NB7d+eXqr^Dv<&#OP$K-mL{VfO+He!-4@ME+YJ|JB$m@1_Y9b#vXe z&-3W)(Vt9?Hja%~U6mitQN<|V>YAqttdCi2#<#ET{=47&(+dmnvI2AyTQGLfM z>#BaFAN>43WU}U$r+nUB9~+V5`QZ3^Zwc__usPX`UujW3Q#LQF==sLWa=iKoN*%lD z87ckTL!is*Hg}u1Z)ayQzI|SEzlel4$6OlWNcYtQBctvYJG)hEfNok*-BK|(DB$4I zNGys)0K8{jndsLDCKqzUFXUO05<4vUi~JG8DYLW&e zykI8qWtJ{5B10@(6MGY8#*&b=6vOxEx~nC3Y&egq8)cD3gXCUnNNtSiQI?`lrrwMK zR*-Wm85TlxsJ2dZgF(PSOaob<+$U6F6sZU{u1*#dV@Xv608M`0qg!=DQ~$I?BS|ye zJE&U@9L=vS_(q((b19JA6jN&GIjLnBT;VZ}9ql%8CA0!$HQQ|y(hvdGx`90y1n@~> z;#5yiKw-W$LI@=N4%NFP*el}mBrS@%KsLDr8*0_BH#GDEH8&=N9%P7uYu=FrjLa{A zQ-p&aQTu%oWD>_-DMlg^QWqmJ>~zHr1w_{-66%wLYX4ceEJ5uF>MF1q&VDwC0Mq zQ7E-5VMs2nse!$ffAjoI?z!jPMO4cO<$8gV`bMWpmOv_Vlz!{gs-8eENVXZZ#)c&A zb~Ip=f*vX~vUpuhU?-q|C=v;z ztd4qPa-$;MRcdxy6Vxb?oszV@xAa~y%NFfvV}s|bDcL+V5I{GKpF+g0mslTAEEhLp)K?JSxddcrTL z2s$B8NbfLRs#h0Vtk^MRK2nD+{%8|y_L2M=mC2gWwev0omjZxwuR{s)+Ch2v1iq{* z*-zMW7eoe!oVDf@6Mu9TG*D~t)yH5jFAP86I zPBVy>k^6j<5<=<1K?2W!@3ND+YkC{1yA`_zds-dZacJbTt2^0=vaua5iOmdDS}Pt2 zlO0y`)!1a6>M1^oJiVPpvl3kc1z8MnKfyU8Xk-PJ~RfPL451gx_>yZq3wpZS!sCjA^!li2>gAIw?J&-nI^Rg8<1;S`_F41=uHb9h1mr z3Fh)q#Z$sW2<%V6%l1p|3LOjFIXwdjt=n-S`X#b-wm}8Op--~dkFYaePB8hvr7vB>r|YL&ic^N^ z4Gukql_Gdm%V1TPihJSsTmc(M#F8HntDKbb1Ftp-CxL3W;wYV(W6@7#Gqr)ReP%VX zn$_6S4h^!zjh!U>D0>>Db4RHTJN8SV2F1aC)TnX!a7|gfX_V)-h($u_VA2#mL#UXN zs11p?R{gKU%&xW7B@I_sd;HWhUsV*UGTl!~&7a(cGYcUNBCRf;hls2Jr$=!Y@J7neRZNS^6MeFP_c@T> zU{abH;X-2{n#Od0^U_g2R&X9~Bn|4NJ%m&N<{5Tytf#EjJ+M|H@Pvk;P60+p~z-?Gf$L|9+j4%UX!Q{QsXLDPb;J#x(zh$W;l@R)%@;>@R~X6Rz&h` zVQX%ZUhX-SB-ag3gkFjr?btiDJuTpVn3_ydzESEG@$%k`K&;gQI1=rPoRHjTtjTDp z!xR$!K^1t;2)phZq?ha@ny^<&wV|G>9qP6~CZ{CCBKc*K#B40UlK*Vwxt4nj#BnV_ z$BA-uYMPFXv*;_XsAf++Ty{iq@NMBQN?6$6%>j%mgm`5=NS`Nup z`_!&C=@LgO#cDosLt5*v?%Y~ntX)%us-&v^kL1Pm%B@p{AlMvIh^@?MyL9b^C5QlF zp&eXtVDUveTYBs%?o%>n4XRc4P>rOY9|!%YPI*{Y9ZA$V%QR8QNu(h)uUG2>0dhK> zleNWWbNF&Izo-L`Q=KoR(W3LWFmD1mk4 zEADe$nRD%8xAuISyY=Y>Py46cEBxd4Lig$WFNCi8k@z)r0lnNoU%7sNp?~#v+SYgZ zm-Y7NpAMUO^-n+lbZgIfwDH-A|MI;Ea2E#B!{udPI9l)Kx1#hED)ZwT|FFN^tSguTE^pJ{bpKK%93M!mV*&FZ%cOmqIre0vt6yQ0QV@As?Qhs`XW;kTPxy}R?genL*_ zA8wp-+3~Hnn)r%eUT=>VuG#hR@)N(-GT8t0DJJ%}wq)&x-_K9yH`~qaa?Rsv;TQh- zKeiu^KmR0N+wEGA^m>tvjp26-^7z-s`qrE0AN|A6|NCj)uD-KZzu$?5^>@$jzFKhX zLI;aO`ND<0**Sgf{(SVc{JbY_H>VF*tH<^3gTrs}@^#+6-k%;EQ~a`_mUpuW+HTf< z_t&>vcER@o)cy*+tsgBr{#y)Cei-;ah^}lD0Gd7z46_7^@sCcY@zPY zhx_{b$Gx!GkHU$s?Uvj9`;Ra8yuiK^6YZPr2fp&xcEvXr4%=qhF3La8;s5VaP_<}E>Ah1Hm6tbE>qAZtlc@D)%Kd zURsTnLTtG2-T$}9U?9-}r8yd)WariXq%tcdAi7o*&Qn4z8b(YXW6MD2Ne1)rsqZ$p zoG>LB!_=GC4s|gb-Kk+z^VF6Lm{24rBylEKr#c5GM0$-OmZs^f`f%n1lmBl)*Dzs6 zu6oePQIw=Bk0_0hs;*~Fr_U6vmlP(ORhIW-s| z|D?AGZal~uQd>9*`C7+N1E3-(o3q zw<&fgVN8XP#wn&8q1bP?E|HR|X&QL>O#9AVA9`d+Q@DS3_GF5 zwa|Y7!wvDE^q@VP!=U zPz~&}*2hHW17(RIdgTBpu1Ut2&;zz8l=nmm%}Iw46*{FMm>?gK#J${*%~n$^H9IOB zsuv%AAhkuz>Zl8}YuH(;aS@;g^|j4(x2pDaS=w39!>$<2CUgkRiIPNb^u^pJWa-uc zW2G{(%#GuahxXbG;)qE7 zE~A#!I0zUD$S?250tKpIK2%3c{C36;)U9NTH<8FyXUoTKPJnzFX>u?ov%>?q9z}@~ zumWvAbj?}`c}c!aNF@>pp^d80$H>;|1c9^#^?p=vE@NrPCF)TkMha+y7$u^B2a}s0 zcwResMU2X07CgxRzBPqvOv`nNHTReaSJ0#&52*$w}}$tdUEo5UGtNhDe1B z`8`9zT$>2&NCY$3g=jS3? z&0~I4i24@ImDa5W(^6ebL*f*6=U!J|97yvAYDvSmr2pCQ6 zWGPJqPqpZ<^o^#fyKL$kK;!LfU<5=Nigql__gJKnL153^(lq)G&!R>}E}W4%am?*; zhITO0q8vw><0LE*fy>QJlSnBtZrh)!`*swEBb@!cxVT}{s3=idN-V!e77qoLy3DsO znVR+Nm<<&16r1!!q9Nj+!?`twKT$td3G9k@;(u=d?7o-ekw76f3#m+fdSWQwVs?&;7CxRAz(2kCP5{uIDrMj7# zOlb;-5rv+GGRzIzp|3Y}Ie@F1)acU)2>U`5zbSToS+Ef4?)Hg5fP{(~`p`J8)iYP% zg_Lc!OdCQ#KO}4^A|H5A_7BuDJ5Mh&6~eV zeNj0}X$;aTy&?v}*4lAhn6FsYih7m`OGDUv_YF%zO%T|6hMZND)D00b_hLgpJd0gF zaY|B$$4Dh@DSmedeCWNEl|C56REjKiF~x=)vTpPe^Jn0*>e;0IJW8U6np7RiIE?xB~mJe{~rX5I%b})6{@M zmtZ)M;FsCzf-!P0SDWZW)F_*YN;5`nQA+Y?vtvqB zT=Y_?U)};|Js4%&oLm!ygw$F#P{W~l_8n^^D$$=wdQ*a&2>BD)MIofFPO-6}J*|$= zSWw4EXCg#^ah#-CM+cl8`elG{$D=$1o=LH;r)z1MbOh=at!ySSmLkj>mQu1P8gZrc z4W6>5rR-u8{Gyk@IG~(tDh6tM3SvEOhN^Rby|Sj1hYSv#CJ)}$334s!4j|4rCVMJq zC6O9CN@YyNC3C=s7E3gu`c!_UvtFVfp^1oV*k-XIc4ZDZda??JR}?lB^(z=?L^<6j z6)v36fC7{z`pF(uRr7_pni&tSA z3wE3N2#I}fQSQf)o1UUp1Tp&yS}1V?wEwmw=o9JbgoYx#o>%s0t?7`%D7FUbEc1qI z*mm;gh$maRevM>sjfd=j8AwdHe2$ShNO2bF@@;*K&5CXxB1)y$2{Sd?esyPC*iQNX zQML~91!_bKQ-Ot)M1}*21PkmYB4qBJf)2Y%a_k#wk|kT;M@&%_;kq#fmiA}Cz7|#5 zkm6q21x%Y)6geX=y}5Q+#20CBGw=0iu+Wl|JBmrLBSM)M&T>Yxoz#y}Y^SzGVIO2s zolqGrBfo^2B6_M6Thv}eO!(^We>?v7m*K>OoBKvz5XSAB_3^l_7*~POKDxjK()O?C zyNa0GAMWPE1yXsK-)-ji$JOm&^SBXgPngo%&AZKZb-O+;IO11#e~opUSO4vNJZ*00 z3)t|2S>CQi(fRXFm#5qv=Z)xUfBuit`gFPT(_#P4A$QyPUiZCwrzH2=X*}b3$-|X);f9)Nwzu$kT7(@Ln2$_61?00t? zVH+LmdU?Uq=5#)N0m|!KqAl3m@(4>nt3n z+x7bjN`67u+7RC!Wu$ja~byVKLWJ33{o-g^N*F4N_&vOp3)hLA51=7;^U0$d;F z_1(Xn*N2lJx^@xanh!n<_LpbG@BVPVwlVjAeneFKr+@tRhj0E^SK=REeaSJ>-`Y#= zA8qKrc}?rAC;5%tWxIdU3)bKM{-fJ{^RRZHxmNrB@loWnSIg_^UB20#-_(DLD)wo% zvH#&oFwL*-=iTiGy5hIH`EdW?qc42)xF5alWnKFw-9J6-cboZ>`&E4Q@0U5Oh^P@edL(4C;x&O4^YTMbTxATQB_jq~eXL4=(KaaL?wjUPA`|-)<+#cbwNEguU z+qqIV>v#3KFFf08aa!M*VL84+u~(4w;rSZ=^qc*`S-TyaYu{gZyVcn6jD3Da887#F z|FB;@txt#j_8C|GJS3IKd#AA#ELThP6|46%4Y{!8&u_M^(^qYjeqr1F;0wj^=Nkv5 zzbyHeb@FnmYM_Q*zqX=^)rBR5hgD|?3 ze^bwS%Msc+0YYf%pB`5aYMw{iaUz`PL@Tr25|6C6Z7LxsyUOe*z2ze*J589N}%YN5i}C^0dB%Bd;6 z(SoZ>b!xH%9&&%CggdMtv!C~ZzWI>qdHLtkBMQoV9xV1NhF2f#M%ZK@jdNj)+| zm_v_cRz6M>Dgwo>v2`_LCK=lWs4c{XQ^1$5?KZ59YFmwCQ>*ql7XcaLe%1Lv`5#q3 zNR(@IL^3htE^P2~*pH{7@osmA+$};p*2YUg8WP=H+}RZN5{7vgm!`b))Q9o2{PICYKidsh|C25k8d zxEV?BY|j+z)HHd+Ff}IOvLlEWzFDBrQ#I{vVK)PSy z3r2+Qnj4JMI|dgDO|}LVVou_(r>|6j~d@b3Bspp%4NVlQ59(s!m6FfoW(Y} zECij9I7jhKYVnS&ary?ipN-wTvni4Z-7KvkPR&pg@@)yi%mY%}JA*!c=5IRM8Sv<2!Fa9ZV z?9AAt2BhcKE;d2o;yubaZ_JIjYQ_o4??feKCk>@eg1tXjD52h{B4)>rcmZL3W%V^d zkX2+O2xU4<{u9`xYLJ+ywx@hU8(?;h&km@*tJF}-VOq@%GUX^Y(o@1J55XK!%3i8p zZ6F>^Eg=$v76oo0)J`z(zceR_zq!SVnLq}@#CMH!PYqaNk763r#O;6|#%CXZ|o1!sV)&3E>uto{U5^^F2}{nk&+LYVgLJ zNlcAx?BMfa|1)4BQ=)mc${_`3@dlmcK_Uy-ahg(S&_FdmHFlp7x3LvrOPM@w$!QV1 zBxe=Ef$DEm6i?WBskgEES06;1PbXR`O5niAd!qp*yP|q`2k1vJKPCofU<;F-j7ZTd zLSxzv%HIY88HiWhA|OgSsO?RhkTfk~dF$}3F^7U^4~AYZPqE7&&Q^#f&8{{?y*`Vg zG)0t@oy!8QkuR)%C&Xw84oSa|k)+&>`pzKs4U_1AsU^jJAtfE`LYU;Nef&YVQG4)+ zMF1V9xj=`R9dc4tCtpnn9wx*S5eD%{+LUY6wXfJKH$@olsaK^3axp>tGSEOXNy`wm zmdb?5LJ@(dM!aM3sTtU*z0)_wJlyoc8x^#4y=rCXr4HkisI4f8wkT6v=7&7(1RXQ- z9;u;QjKhtANHW?MQqYR9uiFYn_t65HhLqA5gaTIdg@e2}Oy8InC5B({3^F?05)K&} zztRUXS+GH+BsET%TMJQzPB=n}Dn)}y5A$lpJ%eHfqW0-w>}_oijf>&G5XHzur-_|@ zQ&V88$Ba!R3f&BYgTFCCl1e=^{;267HSMN8U4z7Fu^+QCJHRL2n}Y0^Hu+RJQYpBJ z_3Dl^PZbNF;*}mJ>aI+qcut6C#GJzPnS;hcv=LjSa^fCGCFuw(YayLuq1#nlTz{Bu zk=d%6&oiW^=gPw%GcQy-Pa=_s#O-V`O&fyM09fVob{M7ZiEJWj4#tjyED|+nGqwTg zN79$bW8;nZWGM26#H58MHnk|$X4Vub!&MuJ4M4W5iBt*Akn`A~

rVsROZ{a8(LJ zo9R+Zbf%i8C^&K=OoaMUnu@)KycLA3$pZy)V13Qc>sdETTAWOGZ>Dz`A%pA$D$3oo z^ExT(%xw3m*FuD~o-p?7ozjLZytoo5-dAgG53?~!Du>#9vjjr9K`afS>9KPUFtm8= zYLatCB6@)fx~6E*-`GB~fDfGku_$B=RHII}h{AKWP=<+)kvPeR!6~umqoX8W5}h8# z9m>T5{kWp_iFL#Wu{`$8Ah8?YUm>nQ&yV9cF7iQJ- z?ryGpqEFAbyowTBuikFw*;y^TXzL`!TfbsRHb7*9YNE_ow~g z_Cx*7TS47Uhx6^9?H_OU>jk*@FxwCIPlCHucto~)ab9|xwYqh=?Hqb}*?P316v;Au;=-<4%plzM2 z_yyp&djBx{2g`ii&5uI!R`6vVnB`fWBzPH`WkmFYquTZd=cKpylYmx0t&*3rCiIOoY` zX}fyoKfm=~o_8XhuITpt`ToHcsy%MK-6sS6xVHDWwV8c+VvYS?GjkcN`h_o`+jED19MrnO_LCf0m6xLS6 z+`{<3yrPZOyj^88qL zvOUtXl@MImruxmiKHJ)R+O9w7rR#bUFjzOZ-@JPvTd%0+r}Jq+hS!lj?eEsMZoWEI zb9UI=Z)}Oancr5ks-1Orw{-~m@j|3+gM9pmb^RG!f85$L)(1V>|6?oQ&kpUk9Y62x zUYL>1fc-L-V715f$kcDKcIq0wrFtX}xiqvR?ur$m(ny3VOsVC1sfp+yT@}dv2xbI< zO&){|g7km|f~kxMI#x-HlseVUj5v`Y%3mh7N71BEerZT@-B9AuIgfxxS9Gdv3?9Tq zhS~*DG`bBzWLL|h*dW%cmOOxKENX^U!hR{Fh)oCiYK5+YW?ZR=wNlizUoas9rb5s_ zs92$yD4JDueVRV~g=fQDafKXt3jn9jh{i^8+oJcM!Gd#9O{7#VQ0jBX-_|><=|L@l z6n&0CH3W)OMvLg0oT*<*9zH=7MCV3H)}+vz6l;1R^}&sf$*Y_YuueUaDoQAp5NZ|< zqo}j(^#np^Js(e9IR)o>jFFK^A%0fER zP=zE;N(fAgIH9S8sxD8mpOZ?pSk0HnbpY6Va5gfMe-^_YF`9_=Gc{ce^(HAbHP>nl z1;8Td?=g%7D3dDHn26U}yoQkI54JySQBxtsqT4di^2zGcEf`1V&q#&$Q8i+KD2cZX z(OF@Ls!{@&E{T7hx(dqFVV|;tzex_v4CsVBq;{I7Ot>_m!W&(4KhSJ@Qk}mfM>1o% z(Y68QZE~5U!bYWGv1%Y`ofHoNDL)ju79oMeiq*)v*78{vW&u`pavQ*jZk={S2dX{~ zB+(g1b*IxNnTq)gFv7j)K2X_v(mT@bk?QHt^j5ir7gYojlTZF^D8iXbY$zNx!>TVq z4%&RRJitzr;sJ9&Ww9ia6_5gPqWWFX50HaajWpnXi1os@0=33>W|UI~8*is)o*E_c zr4ZU-&wMFzXOQ}I6crOzw&I$9PMPqWF3phiV-bnCtL&5?<&8# zGz7rVO0J7|qWf`-$keR!!tO#xdq1`efWCypH$v9Y4wAkF0p}ZwPQ=sC=}+ZkQC+}h z#9T1))72x^DB`+V>$?Ujxttn>wdO@_*%9Z_So&t({j7F&_jZIQSNz2_WDfj*c&8wg z2>D1+`nM6g!j;mNJC=gCL~Ir2bKJYZd(QD($G%UDvOYl9?^8_q-mKw>k!+{CXdgeMjiez zl39gN%z9fCVdntrR;1Vwn^;QfJJ{=rql53IN6OEQmOe{|KB6X$(riLv7%t^AN zpm=<8>?Lx_2v8LpvM66ip@{4+%`-`(Og@~O!e3D1&ZLn< zFK1BTf`#@`M0(BJ2QgQs2545j?u7k=7Q;ryI~6|0K<9Yh2xMOKVgg)5Dp;%)uVQ~*ZEf3Xr{ zQ<|o5Oo+U2?I>*7vSUj?dP-D`65U3NBf?2y!xBb_*09Ls5yIK4r4;+!&G91&eW@I< z1iC|!7*U?(D^o8_Q6h#k=3`7YXV~DN6CK9UEe;}Xif|LC>Jb#vK?22&KjZ-JN7bXr z276wb05rBwq}l>Rf=aSA_<_zdT9e@E^t^pajkun>OBR_!!u|5{W?e+4ce} z1jat521K*0#-Y(=ud7jH)sSM8U)zw+yKk9X?-w4cl+aK<`tf6vPH^#LXGUy#)#~~+ ziw5CQ7SNe+*%DbS(4{Bq)Q_0%F>jPHwNhWs+}swsmf5lTTyYrDPc`>Epz@Bkx^&FI z88%;-uvsYx{!GhZ6eJP0v- zhT}kq9nsxR;5|tVKqmvFSNlAT*f^Rn>S)^8hjw^W=Dyb5~WChG!oO) zMnE<6G^5clku0gfKzYy+&-R|afwgL@e!!VjF=xWIup$N?I+(J$_~^V;XHdol z74;#GNV}edaEIMBsT-mR;&l@{{z@~%zPof_ewwB6fwccY{Igk=(ozL$yh<-hV2sG6 z>Px+_Tn7*FbK4y3D0* zh$gzLG+7Wrb;Mq@5h!3AC!u2DO}wg0L!4wSntZD7)FOC6dpdcq7$EXPls!T|1F1*? z#59SWhl=uIx{2gz$6VSVtA-iUyrABKK|(}5K}~EmP9!9kt3EIp$1cETo6y5s9{=^dhJ?s1pTr0hSgaZbKs`$yT}d` zzm@!(24~=tXa@?ryD>H$cfPjl05lWv>hFI$9QgkW9NHiCFT2Fm>hxapnPLg8xVXQs zRF*&3A5Pbm>heO!+0q1mWq;os?i|v%!0m!X)2sF2w0Y~S#oG_7+r#WwKmYmq{Fl`? zAMSVag~;@~{oQ=HKtA{H=i}<}Tw$^rl)L@=-ARDsr|o+8DXh}rg@?!e?t>t$|G0kI zZ)Y*N{{HtB&}WO0>w|soAJ*sN@o~M|-K@TzPw$1ZRCBTa*7iiFUwp8! z5&ZD}_zPnm&mT5hn z)`AY-t=_J0*P>cpZ}$sZ>B~X{SKKb(QcbnsBv#Y>OJ^SPJ z{lVVv$6wW9Z{OPJ>`tq|-xx^J&mKlp#g&Vmc= zPXTam_qLjZef5EPn77X;@;|Iszxny6xAV^KxzOF#r}^#XAYN(Pl8(t1XZf4?-F|aH zKL5l1Y&X5vfBt@_yOm0L>~lB&#Tl*N%^xj_I+QO6Z(*`!y#(JqI3io~<=bWTIQuTs zFnslH-kmE?_S%MDX!_I2qWCn%)qFnekInCHR`roWeWcy_LP5Pe&F%cmzWvQdgvq*H zxDrK{g-yQ55kS1BTVL;6O!9pLE)WzI?B)_j=Ipy_I2Wt^~DXHLr4ak;%TOog5(n5Ik* zYkr;DQkkOIJSfzB@^nDMo$14soJ)d2^`hDvk)w_x@S^Zbl#?An>c-HqGboK)k$RHI zhD&k5(sq(c%8N?os1T+-Nh=E5EwSx4+9DEqjr`y;Oo&U7>qL@_!!kX0#hyR6Q1^Z9;yJuIPUekLzb9z|i_EMR=N*AfO|lwCtcs6RAfEMMA!U zv<7IF1AV`hv@GqcACWcLM%)I(ss?!+TRSqmB)lwUfX9m zVGmEtwjK(s-7K{fD#!Fw6mk)Amm+Q??t>`e>xSG#y?EmLfl`4Ocu_mg)%L(#NF)I{ zM6(&O9{HV1=qIX=335dm&9*z5a*NwMDb&CQ97mCaA`YSRj!D>>llSS+2ew9tM>v|j zEe*1%o*YX1W0!-R8Pq8aDoIl@QC2CA2#B`%?pZ83n`e(ly+h;(y&#gshOF7WmeR4; z&`Z1((L%+u-!%F|iR~wTat_jwq4Qb&2@w8!+JpJ0MuAfR0W-O(1qKD%}{lp7ImUi6k*07GwIq`dUYRyoha>*RA0VN zl)aL&&Pc&#UzDgqoQ<76NFHmi(UNXfw|5emKZYHXi!Vdmk$s9u&sbzcji85H79m+b zeSqmE+GC}FEjhuSmwHK8p$P=<{SVmMhi31<7jvWh)YLgpzk;YSCY|n(D zyhxwNj?N2n8cU;|y~O1Om5TinqaH?HR|<2$pOzm zl6?@AqsglItV?$WFsw-wUStI9oE)hXF!UisA|xArxJ5E@b6sgG(cqztQ+qmlYAMA4~!MuDPl zB)Hvcw7(`Yu|a~`jYLn4hHOhC&9;08#GnZYTsYGioXQ2lO_B5yPDr;elmrygst{d^ znh2ZDWVdGOTp`3@Qa>bt9~D_O-Pi7}n;+jJy^O@FklaybE;iAluV5(}(Yp_37;_`S zSx*=N;3!6py?``?C{yCB>G^c8N>WIyaHw`|B7O{Vnkk2GK^6gWqmWG#)VZ`B5=vg0 z2iQ?BwQ@MA;htH3l2K4En)A|A_jj!MgtjkA%+1_^oTJ1~z!e&>OMVb`m^QBv*hoEI zfG764y7BhnaFa2pNdlZ-W&b)Xi7A2Hl2F&}rk*oQEeDY#<6)xB(4!+HX5EfLPNjs^ z=mhpoq|%7LGZWP_z)^T>3b>1!BFL!rkws|-;eDYQFgxaz#@KB-rI4ct?`bjidJ9^N z=CD%MDLOO`yvg2i%hZ#k4#h}=E77f-;pCM>OTvCIK?a5%jZltLYMju;{Z}4~QbtKk zN$6deeMGk|OTQR1!|HGt8=QYv`#@XT4F81~(1DpKiPZ)+mzs|yV~J^7Gn-9C&39@p z9hI7Tl~*yY?!_uWk)?D@Mu_5^U7o5oY9zqwqa>zK!%^l8<-n$K3e?{in= ze$V3Rjw7^PjM9>(*AQ0SRvy#Z6$uegy+a*K;0n5qRzut1F2Nvrh<3|K%c%j@=v+_{ z7w!2Z$A(%?g14NIm)OgJqfCz3EPF|0wL=BED4sMdVOlU~+lK`n)eO>91nY5WQ;))5 zlA%Ur7s(1tbXq5|f@I__@J2miVF+rI0kFK;Q>lVcFG?JmQ>JrFOt!-j2r#k?dY@nY z`EQ5l{@;Tp`|5E)aSC@EuGE)e3u5&}$v2ZAY(gP^xq3f4K+-Pwa`~&39kphzR$mM^ z<&&=-=i9x|q8~o`{spsJp}j%~jyLJ@%lgCl@#*wo0rUN^`OE3UwXndKuh>hx@NV9T z{?w6~>MX8MM~5k2t=?}=4>mZ@zpKd6Hy1$dd~r>ehSzZDL# z{^+0I@2?$Ze0&Wh{k09Pz0u=lyA{Ur)l@iJ?&*BD+1F99pC9)2=xc#HA1jIP!Z2IG zhq{s`VE?qO5X9$;iMMx|?iXHGAJN&ljRn*_y@HemGln+!?;k{BYB$)<$73aG7Wn!4 zZoYm-aK@YD^KsnouI>N3e!l1Bscu$(+-*OsQhGgPPY1i#=4risdB*xK8z(`2zgtLJ zAJ%U+mGsnJ;ohG2cv|)2^OeNde8DXrK6y5~^1c1;X7!zeaF=QN^=m}019hL-VC!RC zAeZCS>bQ4qXu*anb@D@niq`!wy)5?2rC%n`t|HX;YxL~9wX@OcQMZDu9>gTPKQBMN zs>?Fo46lbt?{>NAvNJ2w?1z=$zU#xmw#?>uoGX;~v|pvykG1?HOwD(f?P9O`+xgAm ztX(H@ORbZ-r2g@Df7HUbJKs9ttuxr}&Ndoavknu#-Jk12KI|W7Um$fA92e-de{R1` zSA!XcPrvjGp1sl~-rL8vz~}9|`QyF)-hSp&cp(aZSa0o)>$?R2e7UL?!V8=Ddi7xz zmvFo)CD#jWE1m8#A{D@UTwM_1S1UXI{lmk0Q}Ox2-|mHAUwD}Hw=oa;QU!F|Eg$Pn zHUqRjXb6uF^Vase24Jz8Yw>jprG9j-V(!jjcH)7ErDNI~bMVZ;MM*Ssf^wUUS3O-`Q#(#_Gtg;B%DSM6nkZXUw`x8S zuCGudpE&w@mO>X~v@{;r;$jxDTD0#ns5;fQzS8OUlweim0(zDX1=Qw zCr#IQ^?j#4HN3brr0tQQDNk`Eq6-oGHPn=zcaSg_<-vmRIlwm5_adi&hpmKJ4MW8B z61|j>2vlWF)MtS5#RUkd|VmijCsAk4aY90ZIa@#i~IWg5W{T(a=wwY}6y6 z$|YBs0KeA@fJ*Xt;tVcrR-$IS#MVW90C^M!73pPSX<#9p%RKg!ukOVofOOh2S`F_( zO>qyU0gNt{xr`iv(Y9zDkcOyaFYUl5eay^eGTD$c>T)&PYQbivPqHYGv0`XO7=^4u zOQdyuF%p?Ha;v--7zst@4Ix0yEG5)#a%31m07g{{q0nZhh>~~A0@zV`(@eS6DXK!- zDDn=~*is!T+G)|D)O;#xil@?o9if35m(=yqsH)a9l`?unC@kS2#OX+Pa|G)=2%$tY z5J_YUNM$K+pa4Ba5?TY!5*w(}9vSti%#inxpmR!n)W0>qmoiSaJCxiZhIhf{fz4u2 z`#OQo?A#+=da+W7+YCN&u>|lcr00Z-#P;|Y8$u%=bm@>|-MrYErX$Q8y}qekyH8`Q za3XU#iYBwkRM%vtGV)|Ibm}0xOc)09Q!t3IiSHY7&-(U-q^2BXMs-pt+!8Crs17ec zpqg2jh?_tF859jDQd*@(0ilDyj0ro^HLMptwu8!nkW)9Op~rfS9gjM^2=pNe1oDo| zOSjz#+h2kVRA39PTMd2tX5jkJAZFe5lzbV%=%ounw-!|{rb$!s5;#n^=+B5%Uu|FQ zsVy+MrWMcHejsNx3uKOL@S>)s-lyPX+Gt^-1eK9vG!4^$94vCiksLed*o-8YZRZNN zDVW=7aRtU_Q4$YWj~j;;g)`oWb7G2!5La`^&@di$&bO#TgdmN=%2DB|7ib8pXvuFv zZh!*EMhe;AMMZ1K3OIE$T`7zDmFVXzkVm2Tq>!%RT5(UqmLqB%n7c^O%-FjYWji1e z%kGJh-A)$OZRr*kuLDn?8K5yW0wpz&76g(i(?J}N0kBs#529hDsKVBiIbRUF2oDUskuv!hPl}tjXYwYYtpjNa51m*CSnc}X^jYnM9g#AyTU{-}&rk?YLDVW9L#MJGAuzBhS zxJ6QXcoWChz*G(G*vRP=jnI|VYcnA4+@y*%@uv$3jS>XCD5Wq+u|I#N@1z$5ALWsU zqRAovtQp8e%U%jjOr6nap1?D-hLz7m3S_}%%~0S`I%f4RLT-N)w44H;tZsY41xE==Q}C$KinLQkvrtF| zuUwq+57g&iX3hK#aC~})Ab<{#I!~HrgT)^-n*af_g%X}NfZd{Qun{s(a;?;F3f{9F zjzq3&3qKIz(@cRgZ@}hOxdlTLQMqCJ=ICT+@b^$8ga$8|OX-?4zMtAsUWV99K8#Hy zWQ2MFXvnUODP)vmRwRrSA_2vfCkc_m6ncVrj;OT}$Wf(g z)N)s!FlJ)x#XHOcIh{75=%bQ1s2pG)1VEgs*Q0SH+;Z9uD#%jrO&WNX770UuAJLCG z&OZ4?Sa8;9pl-fwhyO^DAZ8w2@CSCc8Yvs***cNxKj-G~g3x|j-gai<@;1}bvitf@ zQH+8u=)EW5U?|P=8=Dfh)~b9As+?+80;}pOW!Y54eHj(uBxvEA7lnRYAC&k1RnfG^#eE{8ajNdyQfbOm z{(_=NZX~vRpBD8^0)AuaL~{-UpT!?W7G0W2w5`=YOCn41db*~m&Trk@7J@{o0IKBd zO97R5wbTi0cSrUVb6p7!pqR8E|8;aq3E~k^e;l;2{^ZriW-=k4A(CSU*cUxYc^?dBIQ z+=}*ln75m!{o!==!qR#7xY-H5xOSq|?dDM!K<6ZV29BP0w-4*xy@*~bYp2D8;?li$ zM%lxs*KpEWdx{Hu^##UvzkfF$b}w@GdN|$gSP^l{<@FaQt935Y-747ODEiXM?<(x5 z_(T0C>)mUZr|$E(zS9+)pjE`EVgz+`^2q^zc29eq&A*-Jl{h|6A8uCvyw~6!D)QAH z>glk5vz;I7fZ99g4SxL9Z#D}rYvmI@{QQ$Zr~2HUM??MNuU23D`tI_rWtQq|R)FfS zzjUPFVcs9^MgJ)EL5<@FHj5$-ncY@yvfgP!%P)5Bq&n`bzl4^E-$kjhi#g8ggnxV+)cp7YKI?PH*> zjgV$j@=(icY6-k~bC}<4oSFDS?OU&9U)Ob3&h@+5Zp9+~xEI9utg&de&l%a*CYI9zIQa}>c6db7r^ZxjP)|o-~Ig4>97~~^Xl%r`rSc~ zSTE(%>uX~Eiw&})ls{$;wz0q6+hViP|Mb3B{*UeY_Rr3;ec2wi40cCt`2}3P*}dWn zzIt9$HoT|3ZG@*~!TZ%;X7{6!vFEry?BAar1WP|}SHGFJ`}4cQ#w>yDoySd`9f73x z!udX5<@z#0k3au(oHedCh(0XhiLMXwFt6`$Vfo4#f}`slj4UQ*0lwFdZ)6zHzH$H6 zM&|g87XN-(lNTCm2f!cBx0ikKR@Ulyq<;L>yNyVSS37%~S^uRi{o}8+Cn}BekH7lz zd6#|v?f1SQ&oY1&O}w-1v_JfLnX+F3?G-6NK$1~ub4N^m_Pr|&kl#E>0uCa%A(TF1 zGhCJ&6)Ft6YB_Z64;HIWNaB^dLS1TNjcO?zbzTRBm~f@8>k4CE$DnE?y7`mc9aHGy z+=cYEdH)@Oc`O>1A_j}s*i<3KYM%v|B{|n%&K(p_%=;EYKQ_6Bger3y^$wCvd2pFI z@PLP8!6n5JS&9t=E3vm9h^5d07Q{Xdj+?wMPdm~ikG*RYOQ!*`D$TwnH+Xg*idZe$IdT8DmUSQukWv5z$l8qL4YxeUDWtF&0h@Ke~^*@K3azm2e}< zjxXirvA*m@iv(k$2iLJ$!mbn_(DRxb!xsI=RlVCyg+0nSL5$u#A z(1qC9z*^=$s^cGyFZnEqa7JY+9rDrzal@g?tTWW((i#C>lo#Jn~{4~E`6Hl{w6 zY(@LQC}$O+zydRY0#oh>W=hU^nkI6MhCIrgz>e~|7RuiaV?$3_bJ$ZyO#LG23!rnZ zO)!qcp+AkGA@J**Ey5Bi9K$E8SrC?7>imK{5>e`MCH+^CQA=hV^#4$%;J3CLo_9PJ~ z69KWj=cb(D2dzbZO*>oNv}!RVcwUKZnbPy6gA%z2=xd3DYr$rmn&P~__Q?F=z4ku6 zh0W?j1s0nl^-U3USBYAdy)f5@T3a*~Un1tf|J(GhdmRb!3ZFdK}SzF-bO>(4FE+BU+1I zjOY^`cAYNUwr{3SN4LL`(6X%`rtP+s zh3Mek&Vo`$cE>z?9>ngy94VIPGEt|>+G<| zqkKJ$WD%4O;pL+(;Q;g4>sqK#WlKIZvKQWSAV&jS;b^DrK<8A>W8f|Y5gSRZ6vf~} ziI@X&r3_*Zv?$hgJb*Y#Fb4rn5usGWL=06?biIT`@(vB`wH+;zW4~nz9Zj$D4K%x| zblT!>{FdGUFS#ci!8oY%8Y!jByG87<1^rF5TH`;9SHTUDhIyvUo6uZcOGfREh{XpD zH<=#Sjy>F@L7)@jy&s}Zd|2@c*f2UVQfku6;G+v)tn>SB2a!-XcP&@ z=46shdv!g?EDB*x!639enU3NxOhV*oT@(Lg5_%Nzi=x=a!m*uuLG)bUG)j4c4mor6 z8tEhV+LExjoa(VVP|7s~SD>JF>8Z51%25gl_Pn$eEF$g(N=-@LXT(FUOlq_vmSKVT z3ng$R+%OWISCM`^5N2d?438s%Gg1)C8c3hq3`XPNZeWo zHta-~bU9{dlbzEd3V{inq?Us~-f;xq$&Qu@mCPoKosdS-97DO9pDvzoW9OhnLZ9hq z6PrNe33)hTdmR&~B8|bgSV+JKQsARg{lN!=bn1{ULZVp&j&blfxY$xbBvR4QmHTL- zq}bR&YSY_sFoPh~=Zr+Cl3Cy>5wm2(<*x2rQs1%Z?NN{}%Ax6~o5D;fX(L#Krml7} z8J#(-B!3kwBx!guYLzA<&`~?EAOTEL#+SxKMHlL4v)IPsR4iqaJU8f)l~oW#~k85E}< z$BuzfL>2JBlTsD96@n6_*orPxGty2LduC~ZfEAi-TakfRbt9lWRutZjBoI`BQvgZN z+@83;;>|6kp76;wTjRG}Dk5gH=@nIcplhjXfb4ac8XMqS$F9^s1DBJ#YxH$%ej^Dd z+b*C^E)Wc$ZDz?r(9lYA1CiE2Bpp!w)1i|Za;oYtPDqh7P9+jaaZ*v82!#7d+l{;& z=fW6~tj3%Tc_+By)y9b361K6LK5pHnsP;fW&R(jtv>K*h$2Z{hMHfqWdE03%iNkdj z?Gkk}43|108fLC~wn+&Ug*q%QrZ&wyiC7*<=Y@)=Ho`0gz{c^d(-=nTz7_^SSb8!X z!gvoXP~BUYPc5f*zD*ruK6D;7$z|USQn`hbXArY6b_aVKL;D^Ufm(zQa~rzQ^2%S1 zDnRK-?lMzYERyYl3`yc6Z#9amo31hrw+4&u2@(=YDULlhQ;8L0XoBF*Wi=D>^r{Mo zdn72*Lo#unS*oF1kS$yjoMSd!ZK_lIZBSZxdiCdDg5dpbz5B4xMn2=pUJJ@!u#?Bt z7xx1G9aqw^e|)o@k6*r~lYNcM6vf5y!bvN(*3({4H$ga8>(zGu{`yupxZ{3X`A3C* zTdxl5r_J5=gM{tZ6~Flclyw~EaennQm6+FQQ0+I5o4?F=(yV{GIUG+Ftoe3+|NJKQ zG`FX-e|7&}0O5thb^kb5Qc%YS>Yk$d-9LV)s9Yx!wx8b4$K%Fc`GUcG+}z#Szizkd z<8eN&9OC&5g7$azi`pHY_V#Wcq4@sI9zzty3oOr>WS<_^|Gqv5U3;7l3yAoFsnkQ& zX{iuKU9D2LR?uR-#@&4RLg&}b)+*7cv6bsZS}!S8KjczzN#cD>Q0Eu^cC|Gxe~nCt7`%sWSjmKLL+8)4-QPNp`1OOV-q}K( zKRELB=l^(juq|;~eX~EjnK#?(Z;!{l!{onM{_9Ks-gjralyaj{rv|ar{AoPx{4DKuN|^2lHljA0>94R zf5bPu)hc`DAQr$@?`do1LH~Mznb%!=ecE3O^s2YLY~W>8zO0L%)}mU)SMx>+|8-w)ORI z_s8SUKOH|>ZhF3NclQsQd3SBA^7?D>PhYC#@kRoOgya2@T zf)_z38Z!4@ZX4?Q74a(&KTWhJLrI2Ui=lzN*Jx!NKlS{(N-h_eUfA9pi}SIRLE%z+ zEpfM?&n)7uM1_;l`H2zGZBH z@^uL*EzMe;X0)gR)C&}aWsq4H8mhNTv^~&>cX%$X&7_eG*2N= zGiQ?eI+B-)v51IB`aYCE;>UI+LBkormQbiv5~4&7lkMBBRJ|W#LBS}<;@Ddx$~1(2 zP#8{aBlac{(z&`t87RfmtNFCy`mU3x)(uMj^x0NWf#6*v*=C^_<}fw3px!^Jr<~^G zGb%!&H)=@ne498s3>Yr4pb?mR4C?=<*4L=~Ks#C?*r=BVFyWTS3~*~|{c+VKf#mPS zE>#GXI*M4e;gea4VMwC!wt%TSLA$WG!iI?g9#BGcx(Aej; zAHY|NEo^C@wx5WM0&7IMxKV=-bRX>G7hgckw1mPC8)vQ_*w)6+8qV+)l1EkCy`w3eSc;tSj3xb(o9g6Fr zZy@V_mm&cKqQD3ekznE=b1@^@9%Z(kjMgxMd)mf4KxopI>iT^m2SG=ux%tQ(m~BkJq}!?+Xy9XV0%DZo)OpF0 zxmlcV*fcp7>Qx=2Y(<7!6!$&!KdQe=c||ZcW|^@sV-f_6oDnHLQ^+woeHR_Eqf1aw zm}GY$_azGpLWw-wNBlC*QVOIjdj+VbMXp9nasP7Ca&j{+(+ zkLDj(G*0KNxy*^%T!y}Dk*IZ+1!pEJF(`3@y)-TtMLBqT;DCjRC~h^hMU%i_D6^!D z26jr9sljLX5j04!gc51$nn;w@Hpk06VqypGYjaoY`?+=-fjy#lyBcEodMyE?NxMT@ zvG68pej%0NAYnCv*{H=ALA^FuNG;wc1zyBmk?#hG=iU%OA7e|A>(4c)y_0cQ9Y&E#G9mqOH;4iSS@_2s8_&9PwirRx+C4IfQG^ib zy);Fj?b;l!#&Y@2vF(ABj26+@S%yLE=ti|&CsJI*zC>}FBfE?oK9nGdML__cm=Qgd zQBsBUxFQo2lnH*(HG~0fvhDh~szvbMw9UDV?YQmPD?UxW-lszyT}Xa7s$4;y?v#6x zfKW7H9v9)$Qk3J0XOC1c$sH0FxAepSuYMH^bLh$Bq7vzjoe41q0+u;<5wtv+?F$n% zj+u3EvS?Zla*5E5>D4m=UW`jig5zO9R$^vmfiu&Gqpf#-|9ox z;8D9uus&gGC=>w*D3VBuYbo>+g;|pV>1sz%0ZhD_d^5s>C-rln^a|l{N43#WGCoNf zm^|np^fCBr*>*9K*sA1;`lWq0D1`@Uak4?Mp+xvN!wWOYuq=+Wo;ny$eX~_y73Ax8GLA z+TV)b^q~scA1j1#eQ$sIaI^Y)K^5C(P7Zpjw>*9P)NkkQRyalT9A@f#_iU|?P_5HwpVo!?fpS7aC}1XtKZDW zCsDKN=XZzw)73MH=wrq7JyfjWO1GT1AFh}8wx`kEG}7Om?RWE4g&oEmU!k7u)mIPm z?P)FG?{0ru-R{pjf3~Of;m?jz7TDE(@3XuUr}^+GR^HQkyO5*q z&-Sgy*W|JSi|X2M?IU|Io40#`KOfJ#&F$u?f~*%%XC0ySZuO^~Q_8O2iUalQ`LNyW z1ev`%Z0`J7zu!O3H~w|4#?{l-X6?Al$QQ#+w)fm_oke-xKW*nPS8wKrjs5Nq``w-W z^^0=(LZw|@U&vwWtyW*8mrHzkC8(>7^nSN+XNUM|QTr&p*D8E|8lhs3w*D$;c749v zXg&|~GuP`4?dSUNm3!F!qt2x*Z>M2?jYPEpaw6V%Gd;hibJ)gEx6adHv-6Q%3CF5O z6cBd3I&SWFz8Q8W-#~&e+YdFx^TU2C4(@-y5F6WCTR+;f-P$egU-+J1R?K1^UZ{zy z0k9w-UOEc$&FZ)7TYK!|Wqx0-Wly+0i~QI&sZES_?#7nFap4kv*xWz(x~faqtc-nr z*~h+))ef;Sn{ar>t=F$QE3vJ_$LFtZH}9O*+jq9(_4mSe&&T?z&s5`Y-)_%N;9LRM z7wqxdwGHw{?FSdC*sw8efip*pVd8Txp`I-T>Y}ByVlcb2viaW00QA#xp z(*+8*h-pEs$x~B7@wKNa8>d8aOH^C)QD)wo47x9|=)8;&qJ_T6V0YgAosn2TbhOkg zDIiO3_Poz;$#EhSjp#%R;<M#>|hhJGqfS^H0?2PFrP|LW3TyCQ<^wWQltF zek1;~1okN*bz@E{Bt!3VKiS0sU_CUHebazbGyjt8rtW3cQL(o{5r1-d;~P&Jy?;HX+;UT{Bbe_H+r6H$wf;A`zo> z{&ba0H@%wBT5Mh&EAvNCdDOhIrpcw2Z^h9-rHdHM-$xx&9WWiic_jlfP_E1YPvAWz zY0*JK+NM0KWf7c?pyC`cALsZiSEvuLmX;4~U z8Y@$wPc&0fnQmWXoJs?2tp!GNS&1}EfFGjYwS74>Lc&gDnTy415ES%k+9TggTy&8C zHSabJXeme4wtzWM3>pm~QZ20&br`k_!D5cF0Y@zniG{zYZN&Nurv`bZ)>Rg?;A4#>y&M%?iiRFJ##sU#{od{Y9JV~^z7PY8#0fMb4 zKrjVPN(p3YnhME9fB?9ssx-3wxeO=xWW}W&pdW7#D;n~K_Bh{al`cdDZ??LO&GA)p z2+EOReO(Z$rub%F++;@!@&`|Na`O%)PG zTI}gi1u!TQY$8mDxSNr9POO+?!gxdUQ4u!KV1C&|3Kmjwm~iE#L3Cc9p>lWXV>+pG zP93ls2)@o>mmv8E(%dM*vO?2t5T7g}qQ&4r;(Jmxwg{9G5`+3*W2;Hx#u`ZEr^Ofp%?Q#wHcfL-)Q#vRHYVi*LiP^ZCA_P=3-WZ1zbw&p#fU@C#r;GY`l(Unn^0fp^gptXpL_M=94sI zJ`&E~LRSg4LZA2~!eq`9AvJ-S5{)9&PJ~saei*a;QoIC*FAq3Y%06QEQu4Ye0Q>A@ z6^RD~Byxr@udjw_vL~H-v!;ebqo(5Br>28X6Gcwykds1WvZa-@hzy7K7|?IDvZ-Hh z>T|=L>#nx5@KO!i=0uH;u&W1gSaTzT*szgJ(_Vx!(&kqcY*(Mulun6fGp4a4Vc%)6 zPq3ja#?mb8n&c5z2Fe$^xFn>*#gPRWx8S5&lm(DHjTr*cVDJnVcy2l5rkl+-k6Ir} zb_VlEko!i(W(aFu?Zr5b?3fqBk|X_E#CyBzV6I8Hg>)m6$`*9 z*D#IdwgP34t;U3=E^0ZVY?1iGRYPu10Ie<32^Jj|ifm+IkBJRd!8=eJ$VC~k0F+cC z9;ldP-mF^hzz^%?+E8na`l4Yg^ig&ITg)Vbx0;J$3Kw7 ztLI)wsayI&*lC-lp)U<$xaOmjYovCuAdR8V^kr!^pxS~>2=aj_QW`U@tZas6M6u

@v{RLZ%>h0M{d!27~{V`m7qq+OF;KS?6a(p*maJ@hN>g{>wf7BQ4T0j2k>XV;8&ZmdH z-}z~MxX>1Bh)(vXqLp1<;mXIyChuV_DCs8ylCIQ&ZyWk$v4rVn_`A!p{rKy@Jyg!s z3qtuBKK$uTUys|r?{^nY+uvVS?nlV>3s$$z@ z-!zl|EPm%oj2*+3*_8#Q>?rV~;H>lN&l|0&)1lHlFBkHQ)`edm{2%}H&p%wJj~`y+ z{{m?Ky+dRd*z{Wg!TTNJ)J#o*L8pG zd9PNsTpYGrLA=iM-Fxa#gEOkZauib{e-H77c9%1w7$Y>kMUkC@DLeB}6-UT-) zYOp_N^ErtuQ2O#DZ7DVwM4=6y(;z}bB`^VDA~~n7P0a+AIulAIifJH*7DK9lj2d)9 z&5wiEo}{*mUY*5>BZ=+S4t*#=%7uXpskuoiL6A2zHU;-O7m6hiU^uC%pXBJ(kc8dR zA-NC*dBzr|6)2?+(za?V8A=vW7DuzxH;#V|kcAebT6|Q9N@^m`z{X`<$#CPEF3Ler zTeRr#8p`BuX}Y3JRf-{wg_e(vZ6t_hDrG1QlcxKVB8y_(txSTKpLHp)DCPcI&5`Z3HK%LTLscGG>fve~+V8OfS{77>c zCwo7vNc5iNfi%8nrg0McZla1nku22&?p=`teM~*brgfewh0*>?3tC!LG^F-X^{OA9wKzuX)V%3>few+ ze9ld^xn|YX^FlozgD`ifDcXxTnzCGK9wEYmn0qXo+v%GMlKp2pumdyuijciIqUxz; z=Qg%Z+1vMRa>XsX(x{{bEgD@CGcR@$^hA!`E3-`1DslG^N+tGooHrz%zABt>l(Hm( zUpH#Voj1r#iHI4yQaqf3Q;@+|Vq@QjJqvV-^u6!IY1^kh-*dMd#TnFVqK!aROg3s>iYcWw~cFDPSWzQ`F3Y zIvF-55vje*9h-<5&diniEXhfbWG-NNhH>nW*jb3I0m^zou29Q*RhQTAXMnlrunv+9N>B}KKv zwwaDPx@I3f-Jy#+pGp5ZNUf$P@M#vzoWOob;XB7yrRNSf6}Yh#!mJ6xv(?PlQR=|v zgNwhS_CpYyoJE@2$;=PYO$m)VH6wCeZp7yWO-H^vr4UUq`XGo6His8ErQqTPv!qE1 zH|>l>dvR)1@FE<<3GyWxA$Z^a>L*7v9H|jdIHtqJ3^Ra0!~w*TU_uIGH0H+<$)E=5 z_!2NrpeNdHYO1wsf$wSCrsXpd95{+Wib8i!F%gVMq@(6fk+~dNTKQxls-4dP7m8$F zJ7QKJq{M(-^n+UEQA#U@AqVJA9-I}EATB|SwusgL9JE$M$EoM8-bvXfq+ApNo|c3J zSet}rU3fC_V9G_ZHj8L61Lag#4xNaH# zs&8&1j~%>4DM^aEk~vlhsofQ*5lDG3m%wZ>qWQ<6YiMt)cQYpP_(b|ffZSphCxf(J z()yv0)}5pVp6zQ<2`7|!lwKHf8`#O%Lt)Ddy4ZKfV-J$6K(1_&Wuv+2M`2kpU=cYA zsRM}Q1l5z2scPiOeRgfP6K%58-x~(Nx_8w!O6*3`A^cRx%-2>gGD>WVw-^fZv%-7@ zvJdJCKu66SjNB7;P%x^3Vx4gi*D`pul13<`@xvVQhU}w7-OAj< z8nyrrp}*Nd*JK7Iz=h2?qGRYWrJh>`U~PcB0Z&SVK=Z8qB z<8NYd9&5r$s|3QM=uAF9ZJ~C8rY*xqF~>;C&%FW~NGwRC)#5y|sli;5CSG)m!nPI)3om`wvg2t5v@0@>IY0&*#IR=MPsaGazvq>)*aQ zZq}>)`J{(_{z58t_40DtuO4Szb|D1~_Kf-E!yoL|5B7Gq;vjtimcC$szn|}&w=_@P zm7PGZM(D2&hUyP?`1}onp^{)q)88^0!(`tq9}kB-E{ie$%6Ql+w|7pT8<2Pvhwy(n zAQfqQdcn8aAct=C^}!b3@nsFxA?~is9}^~4P|P1r>ys_o=Wi^etqbt=xxGH^7R|X?*I5-|2IL~ zuNE%r;S=&XIqY|yS)=jFT}eSv+#)-`Qy9!a1w1UDs}ZiMLA5v@sCCGNqQOnb!iiBhG+2a)FKB}(#T?3`nd>$* zqFtS{Qna+pH9;O)b`!sQ8ixkkT&LFJV@NVd-lA)|0c*W6D7T0yqa;K!VX>vMNeJVT zC98BBr9}LAHuC3aEgny)xR8Bv??1L`~*#a3=Y)p{RriAB@>zRHSpY$rxjLiK4FQ zHD%Z6H(8n}6QwyiApjdZVdo-ir~&3w9a!E;-i`@r8OfR>4$bEgT60y^?TA<~l{_|S zQlH;3CrHyt&@Uzs1+7m-Im%wtRt=#+ecm(`VsbKjno(!NhU;Xr4T9a(VtW>LXVU8WMGQNCBW zV%%|(Egp$3N@!%^p~(~#(XvL|(pCo%Dvy_`1bTI|5yYVcQGq0h5mcd$mx3TUk(o76_Ij}Q zf}|02MAY1k7K78POp%41}U!q z8cltniV@qN2qM>%GX40rPv|7J*v%YoQkRrEcGCmmgPz2}MZ{I34t2yHiCG#c_o8#v z!M=&ewrJ6V5XaPZo6S4HSWKKls9kENy@guTazK)HAOcnd!EBMkOr z1v=zFSe?woRb*al2ho)x`6!hygB3SlzR+(a|+O&e&r9@KA+z|d?~mb7VtoWD$f z2EF-+_&DKf1EotwUkmiSX64P#1Gu$XQvjo7)&wCQ$u86OMxIk~tw^&EqI>-i)C?ly zZg{}W1Wrl8c`Gi~c^6(uO}o9j@Nyq0J`=*WsrS33vsK-maKW6DFc6YZyi2(`EdAY! z@G9Id+&9r_J+4NnHh^mu6t_}qR~D`dRYD|i0dN9A$C^#(BQiAP*)m1#0YHS}E%O+I&-G zzZ;7(UE6?^j*~%yIU^)`qFAACr1$)Uv31?34!Da{)H)5Bo%+1kM><*{3Sjym)J?;@ zU%eF9Ngy{&291oif<4MNc^q zev>_1vTEbV>KMvQG!hDKQpKoCZTsrMSMC5$avL@hzm`(XMDdTEC$`AQc`Hf;Qgjd{ zD@wetQJufoP+wHRB8Xg6lkG^&@5`pJ}Bq?6OYxZ{5QytQbF}Q=~HzyXk%Y$e7;U2-c<4NCU5`-75 zy;6%!?zH;bNhHi*kDPadq6V=zM6PK{d5~hvQN&DSbkda7K?EO|o$Sf|QISE;Uu#)3 zkkmFS+KYm1#*;wL$oDINHu@RKyUXE|af(XUf#+u-DRwyz(mtTBbG05~El#RN1r71o zcbL!?M0&zBZ59(Rb^7+=6Ghq!wNw$WhZ%yV2iSM0IjYq7p;GcIRY38rLkiqtppMmr z)hUvx)@YuvvwmA)^f9L)T@LcT53IFYw2zyl{)RILFuFf9NHm8ibR6^i`(zw?|(ja`2WXHw@Y%;`f^vRk#GF!HNQI@x10Op)vAB`q5bvy!~StT zJ__e}wThRY{MGzX_)>?Q=F1NRxV}If?iPIHSC_oAKJI`0yKlY`lk(%odEVJ=b*cOW z8eN9@1%g-r%_~HIE||~y1!w$yILx>E`_0ZCe*0;8zXcd><0I16&CO}MKG=s}ZQsv_ z&H8Hf-Ttt*2lZ$9{mJHWv%X$^wOeWS&+EqdaJ$JK?O}g-^nnzEt_?Flmyh$m^&Fd7 z!}>gY!iS#6=YD_G!aL0G>qB0z`sc;;;KaqpI*U(3T@k>kzqp>p%s!n3Xtlw%dAZ%( zI##q^;%5Hr3ci*c>|U|Nc2tO>S@=|22Ali!`F$_Z&Fifb%bNLdUfN|PA!zw(6|SG> z_{BFrU;lb@$FJIh^cocWUH$H7(CfaE0xt{n--a2#e6zic)=C@>_99oGL2_^QI}PdK z=~8z6<7d#^CoH-6hL4B!ynSM8{tR)wJnT1i#r^#!{i8U^=lZYQN?`we>m=p=G`5QD z5svfWgH2yWc8djDbAG-0P5pf;-Ibk#f^pA>2fN1X54%jl)LjqHdsaB>-TU=>yZ&bP zbSZm%8{4DS_+rd-quY%`qo3|>Chq!q{rb<%o*qs&&p%1m)8!X7yW5j}CK#}lMzM80wsX#&bJheFB?Wm2M&Fmv`-)D0J1 zK!V_~NH3}B|MTcj(k@gowNR%~4ffN!7ao%GE0Yii;7$dLs6q!xw^8y-lt~wLYfqJX zN2H*mZh?*klYo_sT`67TWL95NZPgT)+sLc!37$B3<`^7lZ#;!CS2;I?*$p;U9Wj4O zK4!vcSNAfhJt%jXnpmq&R^NA|{gEvUDo~M77k|mrz-p^HO+Aqr_W~7A4NmQ=%!r^0 z6_UdQ+lP&CkY2?0E%y6LDl#{`i#1ac;wvCoZFVZVOp5BLYfwtOR0s>bIrmLO(0N<2 zlDf!QGm2UchTcLw6_{b6N~8>7Lcv+XkqlE=ep20)ni1*DNfQOqU|Gw7KwYu}UDME9 zGi~;RL>Z@;`YCo)S`r8h<*zN0?bUIz z;TOjwC>TVDW`qKTdp3>@!H=IWN@7sRHF@fpjuAzIR1#3)fflLLvN9o#kT+<{EA0@H zj|rhgnMq|JM+&(_cwkl=sn15D;}Hg{A>6K~aS>Axok{}IQ3_qKjEI!`Lgh87HoZsh zw31J}E3Y*S zl#H@kPe>?rP$dxwg3O_5YCP3j&aE``o9b0C$#o~kFi}U0=oHX%u@|2j;F7zBc5ftf z5SA~DSaP4lb4Vt0lw2v{v4^o2l`OfYqPRlX{^VTRU>#Jf1G}7^S%>t-#KGJgpj~Z3 z>t15lJ8#r+CfW7}SvUk(yS-2u5OIvj>vV8u@yikvr%E{!WxBC~7vSGAKvPT-k%9 zwNW-PjxDcbJwJkq(6AI#{3V2>q**ffICiL*D~ArW*%+kBE#BLQT)d;C>gewFpGF+F zt0Y|`0txI)>)DF-^7xTGS1cA((3Ha@1z<8+vg2g-E&hv=@dKZKF#AKkT+p$Ovz08O z#ohpE`-x)^9hVNN-n34UhS~hWk(R2;EE#*iLSGalAca;#%<5%+8#w!OU|E7 zZf?#Ow*uzDPXuhV2{CU6u?+gjXclqQ>a9$zh=_Y< z@>?jLPSUmp3l*nH7)msPc5}GLC?FC-8`QLgub2fkYiicjowNW0Y?L|PYH@=fhzpBE zb$bb`HgIVlc{O=K2x?;}`ecB$SRfR*6-j%J=x3wAM$N(GPKrfO^oC-SjXec>f<3oD zv%0ZMCfKWRa_%fE~Rgp9+`k0a&foLjsB%r4=fFeq>v_2&<$mpF8yuO;2ANqw6@g)6)a zr52QSO<;MmTGt6=YC&|$4f}$x5<8`^)*zLw4#f$R+M}p_QM-`hlw#p`v!it-q0KV( zK3I4$5Yi+F>rX4loR$vDEFyes7)m;a3{DL>eSPsP4~&upSqOD%Oc29y!??bf8DTLX z{*9BlY;+`L5ir8!ih5^KRsvq$T;;J*`>URzRJ{rE-8p9h`9P{yVY_Zn(uZ8u-s@=C zX>T@@mau5@dYa%QRVeIq(0pYs=L`eJs1_@TOqv)K?R*_ElN!YyPZOCHoZ}&<=C_O? zk)XY@+~i6NehFJb;)MuR8^v+~9%#1XQfC~z-5Q}TOiok)Ib62+?y|mfYlhJMMpbd(j{zl z%%L5ciboCOGPYdV!Kh9m3X`)rgGdTxyI93JD#)52L5W3eZd#G%$Cn9lzfss`aGbSM zu;7uh8V0GKIz*?Kg0p6{qZ!NBVn@bwu5t$)@`l1|Sap~wqnJZBC%lcwEWA-{jkFX& z&K(*9>A_OUN{VxIUaKgGrbz8b;H`8eS zl8^~~PFz&IYeT(!yYX6*7G^N1*-CtP2}^Gxu~E4_~SC^Mu)?;LEkrB=eoV-lq$P!(C|Bt+$B zi=v1qNJxziMY6L{0VvK=GMeN#WhWvBlkB~J?cZ&CG(^UA(~2zr;?I9RAou@6NHqcU zj*t7pCn52UoBPdneK=$P{!I+;e_V~j^=DAr%h!D6ef(LTAGur6a!~D43Y&F)mHss}c z^zS?z_V4E7(Jr{&UagKdn~E>Ksn7j5-`wpsH|y=^)9@cwFG)%l4C6^(UZB&<*FK+H z|K>*~*`E`ztRG4Kh{$CH1&zP;bscl+I&cCfqa-aoxN3DkWz zueXmCR(;&xY;52@?hik#1l9epd33yRSsrphpkKb&^02q_yV+k}|8lm}P=)h(Z_-ya0Hw(Fen zy1#$9B!%kEw$#r+Ut4?jvF(oe{$acSG*{DL>%4QkwXM(lRlJT*-}fc`jMS~$DwWm4 zkN@XZ+`m8b>)8GFht*rHA6vO*DC~hZH)8w#{RQB+0&$PCqseX0-aBf4ce;OTv#gDL zexxfIg&%hNkGl#;cHH_x==-0){>tI%|8NrA+jhK% z!^3uc<9OmTwz?gVGu?9PF0D*$2cW^L<6W7jKmW!4I5QIar*Y|uEw@MJht!u5^W=7@ zkC^007EBW`$KrLAmYNi@0k_bkG6OM_BuVo%f0yDRT|Z3X`UD(Cj^^h#0+G72`aK~q z$&=`~u+L1*OASV%t4X_wloeG4olss>I8}Q|^rPZvu_uBuCs(^i<|Yd$C|(nug))O# zE!bGp6+mNXiYA(p$-FYaJmij=z4~6&PD+DI;sv55$Fl|U1_iNh9m`tGwbj!omer6y z3@p}X(E+8%leutB$5YLS>z;?|LB{JT@Pi7tscT?^lFgj?10L8*dL+TfPbS_=G-?z z@4m0Z(gszTASMxdvS7w+=mD2KC+t*t-VY z)Dor?4FVa>lBFz&WF{8=!TK^2PYe{&AO|u(i@R-UCk(q4-{qJIE$PK!s7dMcB(8%j zmL7_41-vR_qi}Q1RH}gk#>=){LW*rOo1rVnNh9$yA`KDZ#MTOZpeFhP_?)PedmsW1 z!AKiU&H7;K)2G8)a6{yFACy0Chu9X;2rD>@v8c6`{FaHaF5;zVYS4a^j7i2B9;(RD z(+W=gGzQcyCTV!-XGi8lw9|?*-ASwvSU${xypo(%HwkzSMIoEJ=@q3#a|N(y>w;ti zZBW}gl43_^lmiDTh&hHz|E{z;ygxL0NDM!{^U#F@KF#$=Fl1_!GOTa9z%KNuwa zOE(vAmZ(;;(w&rgSJ#?m1s1idA&_G@jLj+O`^Fv^+6JVAO{jK3R}1yBsk^UGwRljt zf!tZ6 zJL(T;N+Dg9E{Kjvpi_t{0};qh9G6g)oV<`qepbx733worK%%g&%rP4yDK1Kh z;1#y5K|&0u0`B-(${LTZ%tuF#5IZF_SQ9_KY{VkdUXLx^Z+(AzW*kX-o6H?Vk`%XK z1?!(S*psFn#ZE1RX;gOoRERxt%-d!c?u?8_KZ%|V5til!0$#~HDl@^@!ld#T ztRs)wsl2Z4lLSnyDl$lxn5P)|wdJlxl0-JoPz|1NgqHM*FuNF{8eE0{!ZtMtwyV8x zYIJAny7Rs#DpQFXl&TM96c(5PZz9%(!>?CthZPpVkC=~E?W?INm3tJUGAp%c^y(a@ zQmUv_5ePDvqerKlYBdu?P=dL$?ytPI5&BncQBTzKWQT(tGjQ~Yc@+dRf*ACOW-HjV z3(60}A(kZ4lMuoaU~s)U4`1FPTw5$n7~nbBOs1sQtvlKVJR}UM~R9KRhG=l9Giya8XOi|5D>U2O6nU*;WydK ztT}^tj?beWn%u|iJb5%_;`Acyo_NUHK%!o&^o6o<_Sb<>0%rNm&Xf11gObny@W2ie|sW8VW?9K<#sH# zDrnMrSHhsz4o+V{TgxH#u<3$Cxkx!W$pa?dJ99IWdb7!Eb(uEAPpR5MA4H3u&Edn| zE2Rjasw2VsgncVgu96I;Q$L1!R0direHiUvrre0Vp8DS@u_nPGBpFGzkSySbqb~IFsIikka9yN! z7p9()8tRl|Rut(HpzfMnY9u>$8N3ER-nllp9V$H;3BhX-sU4Ia>PY0>c2Vw#CcS2lEiJu@=?>*H72fETW=3LTL+GcGvS~}+$83PKQsH>=hVKLK^8oX%d0uwD5xh~eO zp*^p;R>jxgQ){i3MxClol~hgOFw3=C0WY3(2GEd<7D96r8ANu3nF4$9mK&<3Fmqwn zcXiWqql1c0fT=f^P;h@b1(ec3%-x{AS+eSO2S^Wzgi~c4YKa-unNLOc-fUFH%%hrX zXaNt3FQMVbTlgsA7Y7oTN=A`|lY=8iA~o#98s1I~2$cJmw*+B`wuo&X2vl`6^wf%& zr$}%ErEOx%8|rgw&)SHs`Z5*cEt%S37%;STHenRzvS=zJ9gZURz9VMT;(Y;)K~Yb@ z5D?XDmdxHAz#KM)^nzpWbgh&zQsX|fL)#f1k#fx5ylXhGeb^P%6CCpE5YZe3IxVDp(0DfPzIcxy@^le#xH3WsD{SgF=GvPSO{j>W7U6=VC`qQ5 zP9w8QnK;NrJYOi$9ppN8(4C@omXToqVm?4!n3$zpO&LcOSJl$fzN0)LtocY75Sj6^ zl(`WzE(KId)Ty5`A`E&cp@GZ$CM;@nzxeY{qR#z)H9YMSws)Lw9`^^w>MfwjPv@VV zv3Cvfui^#~K)U)O44=VwFCSJHJzwKX|J~E2F94oTpyMy=lRn*bv%A@z90*vSzU3Ib zokdf7uX_uAtl>P~?H?W-hc-~b&%ikbvp<6^q>J@)#(Ab^Y_@(uI;ey zy*<>YOXgiOx&E8S{lggsd$YfPSnmWU-r9@RIn!5uX=8n|;rw>He``C^?8nN6bal7> z;C#fJgMiStj+&hh2w%Tm{ciI|ANYTFcN3-VmA&EV;J?4w%Cs!3%QqMH;xE^y$D`~_ zo%&xqZAQK3VV=)-znI86xAx_YKQzspJ^229ejb@C8=4T{bA9iX;!BEj))ePX6m zX5E5r82QkHj)sQ`LLNgh(E|Gf8%w}&dkd%p#D2u_5J7~EITjU!{UXeI>a`dEKy8bs zZ!piEx$x$G7!iT*q0D7r8PZ<5lo%?`ipr;JTHE~-$tpFRUNu6onI)nYQ7*|2wWHW6 zEJW?7aY|Teo-7(i389c2FH0#li@~TalqKCeVE?>@Cy=3!k=0r3GP$GFj8Lpi70rJk zcq^)R%VIU`bDfiHB*f4Py=(V@$X;~Mfgr5*-czGG=d0J%WT-7hr5|kR7s6CBh45fD z@&QprzDlUJ8Eu=ZB-nmZoV*c`mMMq|he~zQy(ZX>Q{<5OkENaVI>KVkp*%IUhce`^U00nApN1cX*{}fiW!5h|tQgkIq38jW`#EvNYh0WS_>QEZ3S7IEg zU~B@x0K{HI=H!+IQTh`g)<#4{LRi)@>0xP=+S%0;h%SwCT?h#%$#}|;3UeDkYFD_9 zR^xuDF6Du#W{dn)xR!{5DZiEnxq7gp%P7PuilL=8M_JQ0VWb%mh`z@PhV+${IQsmP zAUV`DzSa?t%mEw-*do*p(hx@1v?9MdmjO_yzJ4cFtkFLSwy9|zC)*?q_|_j;nCG6R zMM@TdfYNsBNLD_K+zugn^`V1vJu77gJ$;iz7JxNZZ15SlSXDjgo6e-J@&Ufs6;Jx!*yLN@$9?AmI)L(Z3IU`CJoB{ zBR7HPx)E}q-YeSe7M{f>8 zIN2#vs4_Z5HD*F}P9msdA;gEY{z}skR@X}60%$b%bE1Yybtht_&jRyKmG}06g)SgE zaT{QJXv*gYGqMxci>JUnWJQGLF!54Az-~D8MkfkRMIjilVk&k>Y@Jlw%!n#R$&5U@ zLaMOzECL@Da$yVc)QkBZCS`JafT}%a?+CsSHH`%|ktMv{oMk@4acWSj)ffb!CukXs z3cJYU^kT;Yq~f5HsUtI-ZJ#iHLE7F8uEQ-$YNNi+g&S;VVf&()qM@mYEpD@pQsONW zu{k!rl#&|cv7aX?_D#BInXM#2Zt_VcI|*7p-$J;eP?1)i_lC6tJjvpcFQ|?NEa6Ml zoW{M_;%c;2d`JYWL;`VhpxR=xe_*cQNDa#(0qdJoS?7`P6me-w$b(>dG7#-C9^^d@ zL@*~xan!HUfrSLb>YzhUCFCFi`WLS^T=&Werv|CrRZic`(g`904%jpxf+g@G9RC?9 z%P~w6(?v*!8VxA-tQOCN9@Lmh#ypaEoSU$y2815O(I=r;L<$F^uilB+@kIkBeFIEn znT6<)@>~r@$l0pI3`E=#4IUa<>B2&Iw=;iG{P&Mx3Dmoth}HHDCRC(L*V_W6CO|~f~Kkw9ue$_ z`qqLoGf(ZT)6|j5P>hl%2kh$XnjpDGk6MM$8fAyk&w^ojb4;N~&<)!aD5jsT(oxM>J+zmB1+Z-ah zUiQ>9Ot045xa_oqw(2h2XxoNI~{If>AM}sB1mBFhDf^l|OL5`A`K?-^)IT6W?cse5T>8mI~ zH1JXYkwh6R)hxw2qW+P~+Gwh|wG*2@B`<NiAIgJ zg~tV;MN-BJs&=+Sx`xWXzs?pxq-i4hYyfINmA~@~C3;57q(_HbnbR^rlA%|iONjv| zy}*KH7@VlB^{#`uxF`TGuJa3LNK$(`>;X}>6W(|j)t-vPMJ|!};)8r`?p%iic40dW(l#L<%OY8OYt-^FZ=dP96uX#nPpdse?=3)ZOOXf=@`|X@30X8I zF+!7FvON_^Ml{%o7aL-0KXxQ<10!SmO#P{Jf(^aO-WPxW^Rc@Boxo)cRj2(^h2Q?{ zxZ2KZ2Pz)wN5T$PIeXEQE{MXb)yIwShpTUY@x4%Z|N2k=)lA^(ZDsCUV4je0;oK{q5amJ6`}1>m}{|ANCKY?LxnL zbF)9~gmly>gz17&*zWw<6V>5c2u;fsU09Fv`4viG3v_ltkG@1aJ9q4QWtUj* z-rL%FJ3oG$=ZY@<&4c#g>f80fHtON3L6Pb^e{iPQl|Oj9{?3zW|9EQ;vvb1Rx1V0n z6knpWUy}J&u)ME^+xbQ)VjGaV_2FUrsjkOmM<3?<^=2otaDkKl{8Duwt{ibJn&0(d zTT#TC)}2j3o#WHv{(f(J`}}l&_ZRjSfA<#`3+fqByl`5N!h3ITKW=Vi*y;dpZMyaR zr^AQ&liu0)be$CY#;^6^AtGDapENHGx$MUQD^{wx|-^~w?v$Oh& zn(FGj;;Kzq?7(sUVDEQVm*4!y2O+&be2y0OhrD`*U^|ljl!sU2`AC>4Wv3bkq|Tl| zlLT|pTQr!H8WIXIibeaDFv{P1ewlux&FMH{W@}+7~Beue%Zb@Jeuw^|Von=-WLRpPo_(%jC z7T+VJye_CaO=B?&K-wO|D83=?ZOejo{};ba8IdecWfFAY9UN)m$a@`BBz*W3rS0UYHXBeI7-0+ zl!#Ml!UQ@gjWm2+!$G=(l!OZ+@&K24elO`VFkfLb*F|#c+Bv}0%VM%z+8k_^QH_=)I{uT5wtKjJO!P%yo89Y({{k&LF%9g1TxrC$kZK> zY_At2*cNIiBN?gLHGBU;ewyGm>=2m?QJTDEt(%yy^3oF5u@@kndJXIs1BwxjlkJm0 zYugpHd(wXs*>s^&k8L~iJ|-lg>S9AJzl<5S7TI5IDya;i5*v6cqYN~WtII*ACz4VG z47>O}nn1!wJtirKq2jOfO1ZOS@c$1#VeCs}?ZCn;-SB;$ zh>d&!slZCUqaA+Qz$9yHfdN*$rPW&KXi`Mlk=BCklA^c|+!6I~HU#AL!Ia9PyJ$U5 zsQVoxnMS*Glx_<)&?Z4u2r%f3QrK27DcwS6%4jp0Um-sn(0Z5`Dn(rIh|CQwJ)u$* zYnXEY{&9$-nsh09E&Y~OM!cRxnUH~Y1Wt%^6ERrJ8t8tBSTHEW)FD)w^rJtQdMatV zbW)cZkd)Hq1TvZGc8_Jd$D6g7)%1-rG!gr2;`du2qD2f3ww-{Q zJ@wF;E$m2R%Xz;k?JsfyNPMUs&|&AM`m3owbX51%%W%4@!dQjuMUwyA(^+ngNy)@g zWaFfiG~|JTNTJ~r7}c{%BogM8p%CUEv0HL|gyBba(MdGJWO$sN2nj7ql3-qY6Unj0 zhIGQGLsN@%ujVT^YiDtyL`6=_0ON!N4?$$$NPW|}|4#NT3;Ry=t)X+sCK7n7eV!Ox zrNj}@yN+sDLQR?>v`MOE@Xa=WxlEF@f_p>}n-f`%S88EYib5wO$I7BN z10r1YEV;%6If^>iDNHasj_D-}Y|4bhN`s^@(X;Le$Zx@Y+X%18bxL!}AX`jMHaSWd z6nKK9RWnkSQ4|2mV8dSMrZ6_$5cSBD6eJD1d7)x`qdcP>I;1e0ix8k>dwW?RxJQYS z;Dd}6KU^|; z+J%8q_}yl^wLhe>5)_{wc)noh{rmk}d(vm_-Dj54-Tv@+_eP*jaj8y^cVd?g!@p{*0>l3|4)9jFwxuMD=X}53o*}8u%zHR4VDGTdLRUoWGk^18 zeq3+1D|_49`FizBv7*kOU)!&?S?`V)T))h8Tp3rLJNIcuANRtOZa!>;Z$1M`|Jo)V zZ*#F)bh9VwTdj3}mW$=s7PanXfJuroYh5#xAjWx4F^m&FjO>-38fkor2?; z|GAE5-G1}BK9)bto8PTByT>=bn0K;2r`yf`!{&JMZT5s@HXE_f7Qfqwsd^z3_t}v> zxVOEaU!E7nd8$6gGjEPh3w(cbI@VEFw?GAqpZB`oaedgI>?U?JYqfuVou?gfzd)C3 zm%{~du`KpR{TMe}TgN`z+j;Fv?)`p$y8xw+cl){SnBy5NoUvzM5|{EBN`GN?mQ{Xx z-j#LZt?ul}53z zL13OG#br`m9hz3bt}+u8=t~;}Ool93P>hjL;%pcrHbKfsMgGY?522wi^++OG_(5s4 zA_aNIpotWYXpEGMCZ9bNmA|kH-kaCKaW-#oY}{l2iMilZJeP=q#3U(-DpPJGU~3p9 znHq?erYwD4a8k3f0EAIh07)*Slecm_7*aBa5R%Rob;hQhs|KCstszcz?3HsPq**Us zEeZmny4J8S*0tsECl-M~jz>GBydwWA6brVZtH4QMI*E`(dilX(_tYfym)e-Z0ZPcJ zwVOnig>j#SnC+x4Z*eo|RdNmJFr1V7KFu`arJ8a9LgV9p9bsS!T zmpo|`LvEPlYS@-Xvjvo!l12$p5Qyy>jLM|g5Tdy(T}I$hC>CcmI$?DJT`iJuon;WE zA`{;*wNnpr`6pxk9DTVG3=wTPxV!BsD*AJD+uuV^y#dX*!Kckj58bE2Ytmi*+1CvW8?C zg98*=dm|AeDE8ao9udN1)1JsQOYmSAD9aYbqXs1$k!}Ynba(Ox8ht7stfBN+X-}h< z`Kb_AYwctn#bE(lGg3*w(d|V)KTw^>j^iZd(u;(k1L2c9>uoL#Ff_WX0@KYLTQ_AH8fyU&+$uQ0-08dEgq;>+Lld(_qG@b z5xeFdq=v3tABY?lYi(KFOhma6)eeKWsE|8qo)YuoL_v}=<{fdV+eRDPa;yedSgA}v zDjhh1kc;Ax=^7&8>dScnna*1m+9nR}LA*^+p9*Rl!-7GU>~Bw}x*c71g3?AW;=`nB zW^Zc_@$Ol|8!unKE+7j=NU%^~VrXJfQFP)uLoY$imqspMTTo%fx*@)-V=vu%BEYiM z&asdhnMq{SHUTa=30lVnW82jdj_BaP@J|-(k}`vZ(}e<|Bq@NAd=oS7)DI#FSNk)q9VdJ#zFLP$TFM7e~6B>?Ja81&1#N=I*R&4P_;o`WayeOr9M|DZE!3+ zIU6?68{60q%^F(HR{H^VhlyREygVUEh{-3bosEjPy{+r6A)R*1WoCuKs!84e(9eka zc4BE}wK|X~w`CndL%-vT#nD4leKo_`=z?njTWBoxbF!#inHo=hytX6Sb~`kbSumjf zV0zvhM|&F}*-aA3f+DNgMadh%YMh2l#eQ|5bxqQ;#xClOBlM@37AEWh^Kv8hAxNpc z1ygpejggS-N@utPtiZz^Myjnx{Q%3t-px(S!$AQLt@zQb7Ir6y>!zosEn9L?JB|iL z$wta{n&q+yrp_=k@(IxBYMM?6$+Li3c?Nv>q=*j%A9_2lnf_LI0Bk>>=*W=RGHS+( zZZ(qbQw-hoivF!Sr!g-r!VJXYCdnD0UU#2Z9;0{WVG4#EB--d)`?Al{%>Yv`{E0B+ zbfClxOp9k{e!#Avpgb_)_w5)LYdZK{=5aRNW8peV6ucj))k3V-1+#%Ju}20$xw@jH zcKcwHaO#v#52N%D!7)ftLTu!!iHd_dFCC)S3|QR~5{t)Rtx)8i#85$o#ezh{X(!Qh z6w1`uF>VJ3GENV}h#g!Ow+nMe(0y~zBa(8~-euasMJ%;oChTYmO|0)`LS_rZRMxvM%GK zW>iSMC-ZoS=x4AauIX#4>AupnB&BZWgLzh%&KO&cbH9neFX*94Bml`zvYPm*RcH!g z7o}fBbv==~jO*tR%Zcs(M7#!qKbD4y&<(H!UPu)$ER6Ds?G%buyU=N3u27ts)cBHz zlm-QCcC|({Nglnt1Xl2fSG|cUjS5F~G=Zz6Y!NkMsqsFMkcypF0{4=iq~Zc38^&>L zt(*@q=wiI7R#~kXH1Dp+LX*|2&b+}BQ9?t`3$psirOVCjeRPGd$26Kb zg4~EZ6!qS~qEmLH5!US-xvZ8dFhXWB!E;ejup>=Wk;--rG}~9N+T|wd@)9?A*XYF7 z)S?_Wq`tBPfCK|Ait1pmoAtB92dux~!&8e~@GTv?mLS15PM@3bO0O=}UM%T|Y{&iw zBilT1JE(2j3F!!I{p2RJb+ep2M@)M}@b+`85*Tp_bLEj+d1;JU{kG% zNrjTov6y$nk^^HuU}jw46N}=&kyV2T$yqq#G}L{E)VEFC)kzT1E;V5$JF1XLs<(~X zXkOL#Ff)!PtF#y~oo%rNCd_RYl#I=(67g#meD$e>&?e11GXjt&RJ}P+P2Ik&K=|U% ze?H{+zagINyY=qVGi=lmv(7L2gnj&dn`i z$HV@||6?a2R|i1;{%-RbK6bs`@8%~QqWQTFk*h$vz5?x@-b0YT)yKQNz1#8h?%nL~ zy8sNg`{Nn@>Y!fxcfkPdUC!V5>a^XhKfO8bPscYu-`i6>J=TxVEKv+&8~`*zbjczB;2+_w~`vPvhX;3nt?8QjS6iUqYaT zFrID7Z!c)JXDx}_)mF%UfA4?&r~g{UQ9rZW?F28@b)7m~DCLK}&G_S|)z)c>fB(@g z{prm+^%q>+$ez}oNr>$;_8friTlJD}f3gWWukqj8n{STy`p2IF!W{&n>?sI0pC%!P zZLEq5h~bn(4IIdmSe)4Xokr}jR&ga#2WFp1g;QRQl3ER`Z>`{r4oHPVQJrwW@T#^p zjBWZPCJ_q7B+yRfv*_Sb5->4e57pLVR#KM&rKxLmApGXNIMb-NY*4gE*cRmtDJf$y z$J7E5YIM<|`gB$#2$|tY7=;Of1)IKr$t%fEw#dIF1Q&TRNni-RLp|mea!3`GqV`c# z(&QNTTG*8B6IE40@vtbO%bxPm=Il*S#P%jOBtlTzcBDu&WS6w3eDx?vJcPsyVz{OF zzb_Hc?NOmKDSp|P$1p;=cn}!A;j%vGGYg5=B&~^t-m1nuF3kz}+%ieX8%dx`-y?2p zR1K#`DZSKBZDq@YeT%mn5-YvFE$#-a=+@DqG`Nlc{Foe&c03~W)%AxRkIwBEXl-}V z{Cvs9v@1fy5h#)v1bbyJk0Ry)!UX1y$PH7GZX{UKL185^eB0b26GIV7ipY|NBpp4l zfG(1Q1!>_l$u3(ML#ubPhiwiXU%Nv>Mn$Q|W<*@6^WOdoYd7UFSd<96O`RS@4%15+ z8TH~+0V)xZSd#^i3Gs;HN5(i&d9sEB49DM&4RaH2c1TlSjF z*JGpAH*3C5@X%qdQzUED37(=zy2ze7#1TuTLsIyT!F!%^)9GH%j3M=O@QdG%kz#sz zY=ju>{2|t8?-ruAeEX~T%_;poiG!KUqpPtu8f-Oppmz5c>u9X0~Z+3`M z{7DB5>6;JA0i*Lc3|&h`-&Kx|#!2%5_F^q$RaHig4SQARb^1sy*C=Ec6i!rdf3IpTgYhYxd+0&%z1U>~bd#H{30 z4$@U8t_C&8A(UD`04V!P{du7LTAW10L*gKlQk2B6AQ~C)wuXd~TaFiex!BzQjiJr! z`71;$&q&sf3-D7A?HScm%oRY`nkHoS<= z9eVHkfS-(=u0_6!s)VTK+lLS+PapM?Bk5yED~1}8Bszn%$hkB3q|jQJZUFj`!J9CA}0 zhKSszIXWdZO4z8{0zSDJcE?xJ{F;BC=?s~K3v38!d{!pa&S7=;T3rO~52|;g-=f$8 zN%YiDT#yfE5R(;2E|{A+Qq^^sBtlCG0O5e!;R<&1G7`!_8H)g}zhG=V6pw@X451Nc z>Rh40314AEhAFCt9VXsfslK7h^ZlcqPr^g0$1jaU5i0@Ah>Gr#1rBHk)y~rD?r6LO zfTq&x|n+Cd@1VQ#E~JKzJH|dlGwul!Xn7Ygd$h=_d>--aheM@Ody>> zK{fS}$6^N-opK710#f5no?QahQ2Igml4{7e_atD35<-z&n@Q|^uVGtq!$;s5Qo%n4 z7L}SriNKK^LtzD5KnPD#J1DU%XWR3b(C7y@>11qsvFcG5v~tr3B1}>;gSav%LZ(CB zB1CnQ!=T8uYtwP%+HTSh$VEP>EuAc$o3D}?X+lT{JWKGNfKaH^ zhlZHgSA#m!k@??3sIF;3T=J;`vJ>t<67Xd(uct{hdK7975wROejzPtW_9X;e!W}(2 zp)jK2bn0VBE7p3Ex|4HP>^2SLRC~k@jfP^s`UIlFqm#Uo1U>Ot$SWXy_}H7T+h}p< zCDo@-ViZEwlaN{V{ydImEJ|%u>+P6RkkL;q>fN`fCI!!Tma~uAhv?N6g|B3_q(DCc z5X*uO0X6X^+MzZmQoIS^beRLE zF_QF_8N|LrEU{AAxv^$-{R&A)kXnHusYarNIf}D}p*ESbt39^4?D}NRt3yv_kpbG2 zUg&ew_RM9HXe}n$l^v##Y4A2$C|#A&hYg(SI5Zi`B^#iIMhc9Ia{uJ0$g&LBf77Xh zx^Hr0ev&HC0{fxHhJd|@%Gj&U(e5u-VF<6WbYgUpUxXy7s>p#uW%D=3Mp3ZZwGyVrdY*SA(rug1VaE;n zBU$9bL{~bt%+@+FmHMP+jG)60xyqyw2Y?b`Cjy2bLXl!(#Vldde(Jw)%@)-EO_j*2 z-9K7abk4YXgo>c#gaCoYua zD3U#n3(U8p41d%lLSSE<{txc07@C-*I#r7ssY?Hedm}jNos!+u{u6VRYiXZY;1HxZ z7*w*QB#=XuC=H{GA*7RRvx%$=X3DmdMWmD;4>}tAi@*Q*aNH+sn8-sPH@nk(D`wQg zd?Ucy8G&{Q&(z-})GO}Jo5Q?zNT>az@BihW?>F16zVcyn+&u0N&PsXynj_ds{+EA# zg8S(PuHmtIdlL7o{dg79^B06{J3L`3m2Y3*Y~R}T1gTqiNd=%gRBF)+h~}fKm;o>pu<8SA~y+G4nlqvknKR*iAX=C-J z57^gto85Z##TYI>dBVspJkgg!pr`rfWY4wV>FHPBZuf83TLF~?-u{KXfnCs9aDTPC zo&WffWcp}BzPo)RvSq#OdUM!TCfVoTU2;yJ*@BDt0kqKG&i4GbtDmoT?|<0;U;`7c z$LHDn(wEr!ZeHJN8UEVtY`^}(uGW?3FP-tP!_`W;eDhGANO%kk{rdfaOs^a6&0*d;QEz3X6=$p{Uylogu`O%e&6!>w59{6W zVSlJd_3gaT9DGJhYZrZ3-JG_MClURwAMF=68p7jwx7Mf7^Zj~%`e6U~-RU@=Y=w;1 zV>_H*)|L3p=G{A6<-hlR_(i^k-LZVruJ+@fZ1R5go5OK!>nmMHx@-lU+xNeAp#JK6 z+qs+D{mFg+-~Mdz+5Y=%0#0@6%>t%jyvVvJ2 zbE;N{{r>)mF1P41tM%>e##f-(wMx5OFVQ~1P$A`a4k4F;6xR2Q<^5`Nc)NRhZM*W; zezaJD?fRok&+bvn>ZyC-w3ETQKiN)rSbw@Oy31-=J+A*)amTek;*))CbNhtcUcHw! z-8mL{yuKLB**56egbmApezm$cLw~g8BD+!-ik{@9Ia>M_s_m&ybNsZswSUv9zMFk_ zZVt{AfAEHaVc@D!u#X1XwN1{F% zEP88H>TBSdL+St-ti~o{1vMY5Yf8WJ;?y0+{j^cU9FRhZPL)UCD5(}hw^d3c>Fzov zm=943qk}wkl2UAwDfZb2gP4ZlEzDM;CQK6D9@d7^)SFXCPLA+$6yc_E>h1=fZPK`|EaXHEOWV+WkH}CbCZKS*pS0ougYWQ5Akb8l zY&(=dJ)_eKF_iVsvb5#oGwK>BTmUB{Yl1J7V^e9u+|*rB^>NkBzol z0`O@N`V{#%#R`ucM#1UK2}WUtqaY)W)=2}%t2v62(km*L#O{#I_M$+WfU5M1X>jrx zO0UHsponsBKWcP=Jqr@lvJa<=xedj3DA}gsn(o+PTskBIvBSGXJNo2GhdTnKsTT+e zCDoK1!Ql$xPJ&A%y$6y9rIIP1BX6}~)$oT^)*QiV0%j!uVWy}K3ZgU`vxr*}tEy>) zR-Ym`K*6k-GIb5ur+N?O{{?LF8kG10Cr>^YfGfmzk9h7uqH(A%Xt$mkf65(!qG5%$d_2L*m0J3W$~vb2#$ zDpDzjK7ki$E8X@~b9|mn^3*}|PZXlDrw%f)$_+%$q{8Tk)3hiAf+eB3N+@FH~~H zg36#Lv6;|87=?!_lu92(qC!ZItco;P%OJ^REOAO+7;(ZDyt6*ZAf^~}J#DO}hRUh> zKqB8I8CsjJAwmhP_lXAw-Zpb9ld;52_5aO{jfdxVB z{v%Rx+T+Fs>|Ixv*~O8LgsJL?!cC^wz`cAUO0tC#J$4@0)sRkU6ao^W7j+CwAWW>T zhlmC5N$rnhQk;ngPJ7i0wX;b_RFMUSLJH~#Y%X=v*f=id-9vJ(GQAP2?aHhaJd6i{ z>EVHe{lo|@*+)7_!1k;8pTZhjKK+Q)d?M`YsduHk7QOVs(2@|h8l*d7zA>LaO$n)e z#DG@_FDW#xT@S)gLD8U~XA9nJ7j#*gQRErWL#n>tagWDZ~ zdd5-vd=$kK_<_c@N(m1#c^{W+OaL7I{Fw-<_4AabX(gzhaME~X3AYfTYa8VL{?goq~yOOQwqWvJ{p942BXHj82N!5w`$1C3v@ z`EHcK{g@0L)5rQPNU~rMdIpBo=Ai-H`*5q~0qe#B^`Q+O>N#woQ-i%)LSLm@$jeIH z9_vLN%1#e45E;10X2S%Yau9)DqNe+7aXNC>W}CYSwdu?r+nGvUw2r%W`|E+tUKI+$ z)X1U}hfGOO%NK2b38}+7iVr&>H(cnNjE-Amb1>;ZDB_1CgmICn!cbBkHwj@%tq(k@ zprkTxd>h>EH69qLWMTifxwZ-Es|RHZu_q)klaL+5iE`tU==JFA5CaYYzGdt&*_-9Z zfjb&RuhF#jRG(My+m}93yjGFTRyIWlYHLqDX(kPFRR`)5x7U~&9K64xbbtVU+V|f6 zh~dUGwrswg15-QRZpv-Q+`xKQ?;-V92$)J{G|KHQbP8HrQApP-d;AG*oE?CDqm5Me zt!mU*=u>|%wcgvG_qBlk~Wdq{;@KI+~&!Bc$CQA#vmrVkQm@Rvso%kzkxv zyBxp)P#GSDE41V&0xQw0nQ%hdN&skp*|E%lq*xJP*G6?d(4&q{k^uu~vuYj*<7Y)q z0Qcy)*6xx;P2R8ObId3b&|CbakYQKzVs0P?HIwetT$F>6s7*m?d7<1JQnb2lEgT-8 z+|?J?NTi=6e@9NxL?V$0=Il508NLSxk@?WZnF65=*f#V1O)20aHAM8zKf4=nj^DhR7hu@;RDT3MMw><*Qhaof)8=0*i++>93JWNmvmNRlMd!RfEW#O;Vp zA_+{>#h-r)wD@ZWB+eD4xDmX#lD`Ui_Hh2>m-~bB_MXJsdUZj8x}X?^&pkYDj*rI* zM7!YcbztcA>Z_w8c#n3s&whXN52wvxf%iU~DiCe8pilqqFTNIRZvSre)#35){^EM| ztH>#&?n0t^JAatB`-l4l`YDdsmjs|;5aQ5&97HyH z@sa`|Z)_0OhwZ1z$GZ%OfBP_RH}kvW^@;~=^>%hhu+x$rE+||VfZ_dqw?5q5eR=|0 zE`LAlKdrZ`ckBDj)>)SCH^-xm$SmI8-KYBGmm~1FU#)NVhcn9WVJ)uR%Gny%Az@KCG`-zw@WQgtvaXIjj%2``y**>;0$W{9*sc`osO{__#kk z&)V~x9nX{YPGruHPY<|^a(%Cx{i)7?#`;Yi>rYRNso!trgFvsF_ck`Smm~OYb8yV% zMu1Gcv#`?w$=WpjkH7lE^XuBb3+A}Lb*N~N42ECetDE)hkAJ$~EDxfEVPD+re5&S0 z{p<78eeqj=Sp8{cvnIB>l@A5{#Hz{@Aikf3J>dh z8>E|+Kfgw2vpcUv2Zyd7Hny@C&e-F8aJ>6*8JM3P9XPD-A8+pV`|avv53{u~Ea?(o z>`!A?6h^gv&DeHd3B1n}dilQR=hrS> zD3_lfQ!C=Qzu7o`+M%|WtjEhTS?2VKZuwD6y#I0am-Fth(X9M_f7*)I_|31^r^my5 zbG7>Mf4SYCv_8ID@5Ezmi}_?<{BH06_VfARQ0Ki3$lfi@X}kJ%eR_Y|*kZ0L?9Es9 zug6EH7T?>Z(UUuSx$oxlNIFfqErSJJ{KAs_c5nOfvF@Sz)(dv|>4l!Q(vN>SY;Hth zZfo-X;V}D#w%=XJs;&NdfBG=5PgkqoeY$_U-}>KwxjudHfB5?JRyWWC>bt*~Z}w(6 zJ~$Dw4exflvaPxOP-ns){d)CN(7TfAYTH$62r|6VSewiZ?_Oy>lr%SRru4Oy4kbw1 zW>F99&CehzxvXxlwvcDBzC>Wj=M}*Tz+;LMhtLzQzHbXvA(1mlofS3LQ}|Wgjw2xf0Tz`8b*EiGZxt{)jwt*%P*Z;1e^x4lMjQ_OODQO~6B1JUdWGb)O& zcCALQnZZQbpfp@;op<@hhN5SQ5PC!=MUf_x)|{(LqVl*>y3KxY17s@&$sPr`D@53t zqzjI`SK`}g4MY%sNrB?6gv}d$j%qLCD4u87y;)ix)RH|Ya-rZ;Axj~+pY%jBs_dyW z)Q|wz^=n4CQQ2EkLld&0*t-<03=U0>{*0yO7w?seCg| zLvGm1+%TqCpc6xo^@Pm4?r=0`XdQJs+X6M4sP-`8@uD7rhB3j{k5FMyNC?9IBLl%C z$PMHjZAUjmhQuiXFF;Tl_BvBfBAYSsvH76Ri zq@MuAqUjxRPy?n6IDyHN$I8Rv~hwBwDKm8(OoJL&=TSWzG0bV%mhIC4gdS z+%`S29qWzAh<+f=3@VDm*bgnKR}Bf6y(1UN_)$-Q@COY~Mm;5EnxL)}M{yAnls-9D zjnHNNg!JHp2tv@49kMh0klSR=A<>Np)Ho2nBS^pzR#36|%Si4d&xN_2#JD|TbD3?1 z=G<8-eK9n$lGQ;O5 zl~oU}=;c*P##3=nftzfUdGTdz$l=^{YHtxp)6Q5)s1un_=YBS&o)mt-VzHQZM5Sw4 z``E-+HEq&IVu#bBYsG513Oi~zf_>pR^qE{3HFzM9xCqG$KE53#qVb~>$Sy{NmXiv8 z1JTO&LWEO?qA(R?Bg8k{$b58_htj-!L@7!lnX+iYznI&M(oi}$i0)CfICQKOpa+_& zY|lmdQ+8fX6Q>-5trbXKPQ*#>$HpWLb9(u#P7A21dV5_{v+gSMT^sszONM8HrxNX8 z4h^Ys^-s;IC8IJH@!g{ghM8?_2kcavrej$g+lZM_9+pFARnl!4V^d6#019Mp1;{|8 zm=w-SXb9t~Q5QcbCS%3+jIByZ^xw3D#TnP=0Q&uVHAYjvy};Lapiw%o+8djGxRk7U zhw@Ffh15(#oVPMjunsOsmZ!k)1MO}bd$O_8`;vj6uXXTT3;a)Q6eKX zGCiF``ht=vaIp8HB>-zBbRcCRekGJx&1xms_pO$D6CbgR6+Ye3_?j!JG8moM}Mu zS$=XAi;^H2uVyjKGMK4MSm?GlQLI2cA!_|^+`r|qVs1JJyUM?${5qZ!p~o=Xsr^wc zR+2}^VH4HS^wMvt_Oh^`rn>Hk)l_{uM9v`!boCX0O&+*TnZ- zS1(p;Y)7{RNg;%&DhR+UW(26vDAuEfQt=XS@DgH?tT08Vk|Q7?59R3S-7E?l;0M`* zDr$#L!rPFGFenR%orqD?(bLN@pRS8UT84}%FDIWz&L0sS;DMDCu_FPV1Ow65M00l< zb(PMksLkak`d{%#kQoT#rpu5~wd2L?M!xv-pAYK&e*$Lqs|x)SJL+e+!mA2YeF6Vm zK&TIg{fCWHQ0iWe4E`u=?7Qv$=7)-(JQHfJ=0DzT-fkWjV5_KFpRTq$-Oq>p>9{(s zw~jm&l<|)b`{QOoqV5Ej{Xx*+o%8Zm9OnITwwJJH|7v--GeqzE_1mr8d2=Jc;?)8^ z{BCVuI?h+CKj?{fkE;*s-Rj%@`@02H`1Ox}`p5NdXJ7cM**^Jhe*DMH?)`kQ@!r0j zZ5Y4Z&YOP}khos&xAwqCpMvk!hnv;cckA~yvi9x!`F7r~zT4k!-V5BkSsq`gO9v&M zfv_*plyxF1092f#_j}`LGi#*rrhP z_T!&!9l*FcgC>7&Q+v1H-fmW3h~N0+h|XVc^va*Uyn6cDZ|!{q9~P2%J1;0rz3%RD zFACwG?@tH&!**VV^Vjy)`;W&T?6SwF%YJ=lkLF8Y^;;VyTaeCn{oBo6*wDXP@9b^% zUoHz*fL8%Q*LK7Gw(_ukYY#f_R$sk;!kzxELTDY-tM{IFqJn-qKYpAmSL*8VqT=KA`1Gy%VRyCivE03P*zS1=Yf)Cr z^ZI_j;J7utZ~WQ~B*&g?+A9nkXJ{~`}z4O)<&3i$VH@0y_e?1>BnT`M2*YtY3 zy8W5#*mu5`XMgU;yUpQt_1RZhe9j*nU?c0X%H~sZs!K8&K2yM0nK_wmwAi9Ynd|Pru zq3x{n=@c<{9W*L5UXz4GF&jKhf|Ub=&w|;E;zr{jMIPcu^3FKuIaMbgu>YxDuStjK z``N-buhY{s{PnsG1XV8Q{ZgVi=LJeiRG&3>D{|-B`I%uOM)7*H0HpHssAL%h z{mL}CcI=_defH8IHINLF37n9&MOeM2hE_8~`WbN?My!Aetu{2&Fr9Zx#ir@P?O*?j z5pcSdCcRMWvSLytB$=JbWx!g%B>pL=vkZ2QB+M?GkCy{#-Ach5g^uk_rV(ozX39&$ zlCET;^w>ad)*EN0M)I$wy9NSTiK?q_KpJoA9#Op-TnO=@A-c|C{z$-_bQ{Xnj^gkp zUe5}9#_X9e`!vClr|6JmR@~l+mQfn)#0rQwQ6D|=beeM))t*GlK5IN7l`3wx1}o4x z8&o1RWE3A;0@{om?9?D`GaHizJ>tfr?^@MUH*`UqkOZxclKdtFbspK&iJ792sIC5Y zCc>m16w1$%%OpWk$*oW_+Dv>O38ZFTkb?+d z)a^h-YBe1^l)<+BZ`+MY-Q)}azBiSrBZRnlrY>0sA%-%=2y9S?AOJKb(X%7TsV&1u zy`+Qi;aHazBA3o1m)W3jWI%YFbpC8%y#_d7QVi*1rbcF?dO@bZ37B7ElDOp%c*nD#w)Sw6ud}->Cw~<0Y4kuM@0|GXk^o=U!ssjonPO1ml8A{vx zYm){l`3)(D5ll)S%A|O+T@#{!L9V~P7m5`riUM;krqZ(3J30qu45m|Hs~YAScMyiCHfSsYLJ+RLKxXupHqVQh4EJd)G36P;k&@AP|Ku)Un}{c1Ksh83Gav!jaOe7m2e`6TC&6 z4Fsniud+4(w49Qe0YVYP&_Fgj$8eR>qOx6!9Wj=@rZ3PwOkkqw7eKe1!P`6D3`e*yBh~QX`UuOGP(mN^mHs7i@$ig195h4Z_VS zY+@s4QeR!kI5JtIriA)_)o(abv!#qr13&kJ#HIseWYh#QBqo?KzbR8{U1WRMvNNem z57KZ-?pokQ-6>M4!S>AL>RO0Cn(Tlf>d(+)UdN_TX<2HkmM00_RWbqv& z^M$02R3mTfMpwJ+JK}*z>a`=!Wt>!^rT4P~q)=sAod5&Tm1Hd|WLs5e0bXwrD3ff7 zdCviDejXJFp?s=DugEfF^Xy5M#AaMtBD6hiJS^-0lvwSypa$Z>wnsy`;6^~_Fp@@1 zQbwUvX4bJvIx0~w0`ZP2CLO$p7_B4OlgIn%i!9~>%1Tlf%YoBiF(^hbYM_93oI(c;mU$B81mr%=6K}M& zs_z%E;nj?mg?h9{Y!guK$}y%O>gX?=|>ZhjLq~;d}$t;mys1{B*4n;W8U_0DmMrGB>I@tP+ zkn|W7N00I1&p!dx``-^#W|tQ5?VbH#|50$rPnWQ|+n@dBa9n@7TD_YEffFq7b0*Ql z<4YRbO9ZkWX+i9*Z&&->41l?X5O^zOfB9cmeHiz}iyT zKDWXB!h8XLod4nSalT$3cIyx8t&P+dXGr5275kGZ zFP4~p_j}FV%cWN_u^LlHe+pH6!5I6qV_jsczg>%3_}825X4Z_`{yElVu{`kZ30r!y zUw)c*s~`XAy*;??!>_le-KPa8dgTag$IX8?-)?3bU=7oPMm}xsZMe_hw4XK?pM9=H zox8EWX>nXRPd03d%k8b@aOQ=6-W`5djidfggSXp@yZ3n?-OeBOj}>>X#r9!yd#ca- za_BDB%@a)i;%^T2ewwbAFZIpo{g3~F`86GhH>vfhhl+|cDw!l`QuP+ni~=)tM4u+rX($|I zQGozq3<^XeoL;9738WNU@pa&{96b7n^B%xutTS^f3b;Ny=Qt zsZVKyM6MVI(ukrtAyU;yr6J@1QdhcURw2XUDFuvgl&41Uq7p()s;Gv%qE9C1b?imE zM)?78Q#2Hd?$Hb4MPTVDa|XW;v&2ff7@$VlLwmJPDS$5Eu< znEJ676dy5KXO(=Lo0LhVECe-{9-C`Z6uy{n_)?o3NPb(yAQ1DZ#lFL0*&7F{(N-=QM&!8wvf9YFnr z-EJ_vU8R|m!D_E0sv_hLOTu^(DYaa)hU=pGa)PcP8)TnD0Z4+AD@yjZokKBOy&{dM z3|;#sBGf^}P3`jmoA||qRfrO$h(?ggHs&V8NLZgiEak{aJ$VL{pgx0e7X*UrMX)j= zH)<5$JAt%a1*|bB`b7G9J<(D1L4_kkG}!?=QO0q|f)G-=d{9$2_NOaD$~kBeHld`W z_Im>&^a(3{ATjHqXB1FhN+7Kb(e#c88Wm9@P(NjhImU+V-M2%QF-Pi((g-cPofCx> zIftWTqqu2H%xy$llq`ZlI<;h8W0Pg}Q8znBeD?BB8t7N`Ce3p~0(=We8nObd1q+@U zjmNGsTM_xv|4rSyEJt!(YnrbTrU@l80Ofo8{bYx#Bf5j#VE8aJgrT3L-B|-8dPuqC}Qv&IzSjKLW>#o$YM6Iw@(gf zk3?LhBtjW4D@u1FMw37w;Kc<{CQn1O?KyNX)er?4BD`I1(-%;~&3nSg$Lq^7ld4P; zVo?=wm9!lbAy)EYv7DpWEo-Pr%cNKRn_f})S`6}wApYZA_>tb6({#E74ffMznKE znwV5Rv0a6Fx#l0)?!lfvTPc`Ev6EWO*aY%d$qrjPf*E*ZXH@;VCo1)KX9flv*PVR-&Ny9vgDOqPhSnw^ONm z8^TG^XS3jX-wKaL1KUbH8BWQlQa2RZrJdS0n2fSJL}JmDr%c?X()>jje`o|(fN9nD zE83gAWEu#xQ;J!-r<9u34R99zwY?Hoe#4~VrB3#YrrNi&fRsdv+-l%y<49GlMnz&X%elJC zl5$b_4SsAH#Qa3YuPHJ>hEJm?q^SX2@TZKE)5oh8pE}CP$p=J!8 zc6MhBhJGH|`s(P!we>6M3bM%J9LRsqOjw-moDl@_itPjz6<$5TBMGTF6*uEb2Q}b(Y~>Q z$Jz*eQ@1(nAW>l25<`I`MS`^Q)YVf08L!J)R#}7gWpZ?zz!^$4evpk4Gk{30Y5CP5 zB4)T!#tU(!%KDe=VVDQrfzdG^wJnf@rS8-sg*ef=X|pl&8Pf?xN3#?&)Y&$R-y3b8nX7{HV0^4C&=(8bP*-8gVgbz`s%3{ zT|^|%hOa|jgrhKx>ZOxNDfnbjo;|vdU7NRgq(q^FMX7hUWMr%4kZNjKmc9*^5_J5N zK2{|p3FXqGI_V&sDI3he+Ma3^3&Z3((E!jWNEx-uG@8;Tu{rEA0V6-vEJE+eNGY>O z9!_xOU1|u5D)^xKS=mdj2F`y`BV-KNzTwPrW|q($oGO6ISEZ~0#*xYf_Sli9pe1Bv z7=^2>FgQU~YSqDD5>=)l1cXg5DkMokt_8IKjfX80OzBodM$5KjWh_dkJ_NyTAoVVW zH9NEsAS-!Ls^(Qnq5+b^C9RPGp>Hy9k1t+Thae?n#?Sou{_Nv_960+w0%!L1b~W7G zB=XKhG5#60D^LU7RfB-fLL>)md8Y-t`t7>)|l0=lkpZ^HY2hllZWk9z`bnOl_)Zdp+B%+&|1aP2J}i-1I}(?H}jk zg2%lOkaD`+rU@N(>$~YNt`9!rKd!zU@5EDVW4le1#(%cAxgK{bbA754jp?g7z5c8D zxS8!rFPO#Qup3YC#S<3O=jp^q`aFBXf)f^6`tpR9{`158U+lYI+O$n}&)=qLn%1^Z z(y!7N607Y($GiOI)v!Nko)V7t?Yt8o?qS^Ch%ouxu)A4^V(qr;&9L83>(#aJ-U~d| zA9DW!=-mtlVP2h-*@36FOMbEu*^Tza!#wSLMo(9Id7NkR=lkjAX7h|o{g4Iv;Iq6y zrFY{AUOlW2(_=#R+PZic4_e0OqxVk<@#`;q`Fg$=26pw^gdg{}UyVNro%*{ATOjX$ zdH=uLY+QbQuzfft1nTprt2a-(^IzMWKvYW3aSw7DC0_p{C4*^W75Pu@S>Kit`$*Y=KPVQksniA4KlBfc!XZ-(trYulDy zeYN_0yMWM-dtVS6d!GAY_oHl-u7CbWLX8ji5Btma|GIh7zIK@X37VeRgKb>3N(P6- z+wH%YhFg33@$s|0cB-$8??p?k1vLM-GP@^(xV5ap6e*U??rr^lylWBY5w0`RYp2(`_hok*ci$ZYx!+bM5y%>hQ+3S6>)812X812EQ z{jX-hb%6kaFe$M^O(jyP3#hLlU$3U)SB`5#F;ipC0_p~|soxGFHubhrZbRgd#G)jQ zBfAOo47KKgW_2n*eM6ONN$R{-c$xJE$`6zhQ^o!k|7 zB_z%yo!w9cL>22cEB~hWwJVhhL#aQ(u8@+%LX$evFG;R@EtVxp9`p@GQubjWcwp04 zeMfplZ4=s-pd*6%Wu1U2(u@Nr8!LYtTuo`Q7s2=v}a>Kn7&N~#l1UE)EA6f}cDC?Y#u zBo_~Fp>hlsidxA=p)$A#(?M!FENdi3pi^TL>Dx-2(~KCrVk_|^+BS_-jsQr({*a^p zl0no4p+;-bhvZKs|0iU0oIdWAlurrnS%nrcnvyEVo|{S-C1R$rqZSQ`ttngA#1k(% znHoDOlu2y}k|-TIWT&=a>JimYN*^jKdL+vrMFT8g)Tn}2^{Ty6jmoanXT0w|zEZ`e zBw(%GMv_@}o*|P>-v&4V^@ozEv2DZ)Vv?zBHUhCA+vdp19DbCVcY!_qjky;U(QZYX zy9SY%-u6e-p=#Buk*OzE=BXff+`U0e=!=A(8h4`T31~>vdjSDZT?uyvE}yi~QK`_3 zNQVmSV((?8z$TPXr|m*r(fMZcA^GhF6|zN$95tePWDx!#Lt5M5VRK*?G?nP)yP(%slWk~yUa0@JrxG`6L z21l{@M2KnYuBs_@A;!m^fPbP+rh-XT_ozBpONTLol^33eD7aEpIa~wb*tUWT1l?3Z zb8+3GuS=vIQW#(25#pjTMJ9@nCb3c2M2IqR&r%&kxsZ%2O%EpGDsm}oJ{SWgMlzBz z5V@``IU*Aj+dU1J*|~@cNz(Nu)ODw>U_L(c908=i7;5v8VB%K+e)q4Fdso;yl|3Kauy2`ge`MCVDehp25a)NGLg6h z&HS5(K*phnez1p1yXgw^qJIV+Hr**8SA-SpyQ|4aZCu!MxY1Oe6T243bOfQ zm1@^O;#+0w$g}TMkSEw-5YV)uR;uWorj#vgnPAaE>lm_3T76XuYr?46;7vue;C1R$ zD4QkzFY*XvWUzBniG?oXoJ%+9)hj@qa+LNW<0MQ|-H2F^##ppc$blf8AT^nQyWXKhhmZ&6sz3?ATwUT-wzwu{ z_Sbi*qHIi+c8)+oY?ka0MQ@IRA^*h|qym!sbLBlD>1n&PL_vYjIR*wMDyWYbwR*(q z2e=23gY^=i$*(2E5>ecmk?*qhtiqE(d5e(ud#koXsH9rnDPd#vMA-J>fHZQD%n=Hi zfLgjqQz@$t9+IuA3=8VhVSy=STm)Uu+v%4xG*%h4od3m~q$W$Dq{zGx!6Zf>9rDmS zB{@|(<@Gf@$DkDZ%Jk2HpofTf5l-3H2R}+dNlb%RU zRnW`ltIkHT#vb{~qV|%GQ3rGq6i(~PhLUiI!b_nBRac7Xg~$=Bwk(;vFSgXuT1k$a z8mq-$EbpI+on|G-pp~wiN2(%ii0e_lwC-j zJoz6=ofsIzRaM*MYDsCA3a3p$vQl-7VyH_yvS4eAmnTpBr2xb{F^Im*)T*}>Dn06E zfn(9Uj<&?3*N*s}2zc!l7MC#hyJJn^%#vztQ^T31^F3tbQEA6p6xpEeN7pkQo_eS% z>N~YiwNps~&Oe!3V3VlbqLbR7NX{|H(IG%e1frZq5!kNH1Ooh7YaI^QFX^qjV4VG6 zKuT{Nj7Tx(jpwYh-Afi3^Oxd%CdWsfQd3YZwiO9kLrKS~7LNz>zquy^$ODUU8ap{4 zDQKe^vf8!AOrmgP(beu|mpc3SS3!FJ=L4lViR|V^c+dovw0~LXdK0qqusrpG=6gNg zzyIZC8ZQM)egA(9)83xxtw5{^8yL(`U#-4>0S$jSKaRWY{kT26z?U!V$xk@H^aeH% z%fl_u&1WR@4}$!yzMs~2<7OI+Z+# zY?N2~NmRJ|)wk1LnAqcZNUP}mzuE%7TYWbTZ=XlxVVJf;?*2vC~Vdh{=9j*f0%~np?OSbczx@3r`y_ePP_H6 zAN683yY_e+L3d9))cd>n-TvYX)B8?Ot-svvY|2*xcW*8h)@0|{eKqcG-~ZcA52fGx zzFHQfRJfj(+y`vN+hv@5DE#?%t8eE0_;5FE$JHwv)7{nTji`JtK6<<^K zKRgfY#p>H(w;4rIJB?bNclOD&v4{L(cobguo8!%s5ZhX_&$IBQkoj+qo7ETNaC5c* zUk~<-^D{E|?9Og$mN{P*#xm{ap#HNJpa(nJ^Vmi2L?t{eZ~hnE;QQG&_ecBnnXr1< z0xui)js3srX1+Le2jt-4v(?^L&CC9Ky&HahT76~4&@IR;c>N1ewaB#}E)U}%RycP$yS>!ISAc8-kj{3*}-S>?xo9OJsqBw z!GALy)GcvL@cdr`5pL_Yp~$9a%`tY# zQkvxxX>Rc=6BCWtP>49EEN%A^mw0FImnks%9cp!N(rZYlGh-@F(-9hEnGD(t@P10R zidiLp@mRIFnvdt=s>p()ZHS;FlIqv??nE=xDhrA|wxS*>kf=bRGeUJ635ftG*49JY zu>N9c>jdfx(b)i~Uf}V3Ndzt`1k4)(x$bD%i3@md+@_Qw(#R))a|7 zHS1XM35}hHC?M*GOuj{;=aAqyZexSxRh3mT#)6cMX^7}c^&tXlG-4qn^g(Q6hVF11 z9zA_Sut@=XpiC(_!4|o~OOaBx1Yfa3O5C`xtZHUP?X~?8n<=66notosrXX+BmC{7V z?N^z}Cy-lhMhxpYB9ABuDPmTnH!BKTun}aA8IQ#rikvmB2nSZ7H|tX`1-`li{*`oV^G~SER0_ifXQ$-jk&8h@#DpJIlMBN2 z9kN~`ah6~V9c6>1mtavE6}hTL%BfVE>C9UuNNcAgIVFUna0!5w=%U&ZD55j>lE@YM zAj(9fkaG@6>R@sLMsmk%@pEJ^qPwW;2_gGpFYZY%vMocEQG@x4tE;TP$j`{C&QR#7 z+$)Z6!cK~9xy@362)rJ__7+aj~rsm7xRW)f;?2qUD^ys3x{ zYVsnnHWR#?lbDhlk$Mn;j%eJG)6~qrcroD^Z3FZA1!c}lR2vL-xWi9EV$cAOxavFR z#+EJu2~wzCvn8l@YaRzVvT6VbAJ;XD#Py$nTs=mJb;iE!lio?eoSF$bJ981pD^}?i za~o0#t&suCrd&O=aQ{{=Du4m8h(;OHD92lHE+hj7B?h)TJiByc0ic8-67PuwVVi-u^?7G z&STkj0(DWti1%nwc-}|h%qV1WngNtlP(LCwFUaR&rR-i!npEz!3z2-comlb+>0ML_ zrd*YxLR*xiejz?}NUfQDvS6R)vKBPD1}1wc>U1bzpw`38KBh!RQ4|@D#1C1eEvVEk zTHQ!0cgg~{r1__gVaop4&1>9UB0%g=G)|#`d63X-D@ANnh_Lvh&Xk7S7ghH^31Ukp zrNWLGj}HLfDI(Ksu9m%BFb28o1Y-f_^jhpF`5<~%7kIn25Mzssb3(PE(2p=~8T>Bo zt1jkzq06k7b=Eq&Orjblu??havqtL6_NrB+ULuodQ1@!){Pb7Reqb-jEZ%KUGayx# zdg0-*z+%2-OKwTse?p)`t-Gwiq?{_zVp>?51%^dxj--yWx{=a6fP^~jYH~bkX(>@W z(BCQu`%RE16pj`8s9NB&UQ&hVvjmtxfIL zG*RYO*BxvI$!3&UJW;rqN1sczB)8fEKqTbpRa1ilOPrMzkwK};g`)JT7tb!Z;o@^B zGNk@xTB8ND(x_n_g$F8u2s2&*w`Y}u`(S@`CCgc=SD%O7y!KwHds;!Fq=t1Z4TXwS zRNFc-qirR27G#c<6i3%GL4(5`vgQK6p4gg^y5539Cs(55K@H#NqxVh7gT{1T+FyvN z!7NIZQ_K1aGmn7~0aaA%6=8}i={Vw`<&Wz-tKi&5R8l+A$}{J<>ZqSnv>aI!lpG>E z9*{tyaPoXit%QOJlhuG42v?wJt|2slo$-T=qWW6hD=bDX zj9z(X>R?c}U#5Lnc*vx*C|0GF`~|mBa;TPVXmTmL@BHU8W7$a7Duoh*)qT z*7k#I5R#E>l}5(^Fmy=Q=Uyeej2&9cT4o1qY|cLY$AQBMqL}G*_XrqVX&VZ6FKa-yB+T5)o)K9&#N2JwC;YM ze*Sr|UkF9|aJ=43`@3|H6Tot12Sahf^$H=lUyXYa;ihleAHJFIhV6Fs)%djCOxvr~ z50kKU_Vw+>auvayug5Xr`9!?Bdq$!z=)&dJ!s$)ho1+Ma69)5czW49$m;cpg@8;d+ z<}+P#f4EE_)_1$fQKUljJAuv z#p?CE+m7OIy;!{&R)3tghhf^{stb0gm?zGw+Fr_%ybkuZNrWzuZsuw`Z*PGmUQOJ|fOt z-goz8|MGTx;S=7CkMrhH*!G?7kmh}O`~H_SL;GPfZ}-0!AYAM2M$bDeq}G3(H$w2* z3w|>W+ski;^}ihLEx(_|{;CoBZn774i1oLFz`Dnu{F=+huk1Vb;!(B-U(l9cB>ZR2 z{F|Qj-7*%w(){U;*MAoWvlh&0*`*!#*#Z(4aDG{F=XluDjJz6-H{&L~#1Dt}zdYFf z(9>=gz0oqg8TaembiLeiKd&b(Yo|e8wzw@k2j|;N-rL^!z+&k) zqfN!rN`0Gl|37b^ZN~?l6ZV-f<;&iEp3!fPn#I40bo{~}Q49I7K>Sl<#EG%_WMtC4 z%^2I@jX$L)706vvCjk2-vT9quwlIBN-KB5ugfCw%{ihQP`mI6Oc2w>0W!Ml@MnNM%kt~)dG8kLR z_d9bktEZ<@gMXeomzn~4JDc=;-(Rptz_~PQN=})v#~AI zgBzgSd6kKLr6b1dSw%cay(nwx&~?rBtffEYI|@Rm3!ZMzMcE;V!1mKmdshN}(UznLIFcf%)G)jBD;r8RokIa>qLACzNDSTJtWD4ILk75 zOg~+FVX5I^?U^!~o9ar^V>atlz0~ux$XX+InyfwDKetmU!>*>>o0u}l_wA~_L@I;I zY!GwGHsUiQSYFwf>-`D1gpEC&W>se|2~-2M-MYwX_Js?}MWiEb1d0L2!iy{!*J9y9 z?pmeD*6G}z5R3SUl`nUtfeTo>waXtukx9Jf+}cCvmK>{04aMrq+^e#md+Y1)UwWAdufw2xLVbxvE5cmM`~xt_lv(?PX||6xHZB z$aqvzDC9>4f`Y~7l@$a1v1~*2u-pYo>N3`8H3^!lh5lzLHfHE2koqsS0*MKfXRCmBN?aN5&rB$ouCh|>75!Lm)Sfd(FNz0+B)GnUCh)y3$ z&zbv@vAm~&mkd*hC?IL;WEd|u=&fU~$#d(aBTa6u9dVGMD;&xOZyp@RMw_L#TW8Ko z;ot_l3$|2<36+jl0Z~E;&?Iq&iD;4rOgw5u#!TjChhxGh-w=f}fM-^7NIvswq)Ma0!~eZjXg7_ro}8k;E&e!Ffo8u!roD%rL2n9C9JXD zj(YHS&8_VtA+zjEGm`@#e#r>g15@NDwp|jVDFrg_IL?W6!N8X&JqU=?OXXFud03+2 zgriUpK+cza8>J=81JHgX9D^L&0{saW$sst9!$66Ny>yT>5m8F5fv|c_8Cr~!Z4i4V z=PcFeNNqEXlI;jMYRewzXloBW^Q3MlgvbMwh_bB_T4kHIDSK>qv!%s~&>r@J*VRiA zjhO1}&1zbqjmH2m<7HD~!-{mlsBOAaCL5x}DA^b6pi-Z6m*KViFTE7TNG(Ndj)*6< ztz088wu}nZATv>$?bev`hS8iJXo=4WLoXrD(v&BmHPl9hgKA* zSeoiW?gpe0iFgQ!aXF`7SrVYC7C@L7YArZqWVcadHheK0!r}?yzNNlvvr-sgbV99> z1dd%E6{TWdIB+fZzXXb;a-?~YWri+YPIBpB!ZfM6R#4!qh9XD>#V*&0O~IQoT@+Xa zJDQM##QXh_ebt#NkU!WjN0Eqm=@IpE6jU_GNTK!+IJ2Z{SV489bSbvnxG7Jg*qDe^ z%@?kbgWpzkD8PnU)Fh6V{PR}i#V}Ull2Vf|#?uyRL8{gXY%9d*ihBZ!y|p0InE_9S zX%#DC39{v@{za_LNKk-$kES%|jyMSHXtvD*=_sk*B6EQKB$G^VH^R=~f@*?t7Yd~B z=_Lq3OGu3f4Z<3>F|_eE8kAHM{(qy zP^YiwjX=W}tIxMLLI-|1-PnI5lHb{`>71U!`uXqQ4x8O@{7E=Ofelx$hx=nf+S)(f zis<#91)BI`_81~&@gY|PtpGbp!793q|H(%RZCWQWe+-!tF zzka5*-VE=4pZ;{Q8Xs+OkC)4Tt%q#_mOo#@*MNw^?*_a0^f*3`xGkmmou1kD&IzKw z`)qGd^Js7QAlBVwcE1%*t^Q@1y^Xd;i01}lSnbz$^X!=Qhk5h#a5rz0r8r0R4~H`# z@73z_`_riXWj7xmg+q7X{MI(2J}tyg`AqFXEQ=$$XJjAH zfMwtSe`1TdJ|PAlGc{HZhHI)NIq8`Z=R{Oe zjb$ z3ILdiCZ%L%9Wu;;hrDJL0@YF4-j%_@VQB3zfCXx&bIe?Wl%kM!8x5}G_K+0H4x=g) z5=hy(zRHqHscv8=3%N4hMg#|r9R=71q6VCt8N%HOv)`N^YMlWHfE%TVy{Ud)nW2sa zQV!d3TSD7$>?A)9t3iF;_KviEng-mWzRYCEQo>A9vGj*fcc`S=ma(Lor}8*m3+&{b zE%oB#&o7gM)1pgM*W&dC#De>34JmX5F)R>OtmqIMR8JR|JL$YNjdTTQNjF_Y46w*o z3fzE7E>do(6Z5V@~ty4S6V9a{)lJ4lZgUn zLtUIDmELt?e1grnww)UmX>(~mXeb(n+H#Ee$a|4@mU8r(Tmfr!zul0XK92?8W!?&^|3 z&{+;`j*^f?2%nIU)MO(uE@C`VvW1z2D#>jS=TBv4Sp(@ywLk|49F{_f~YhxA#2_u6feQ6SF^+{!sO?f4E zq^&A!EE890pyO2F*cNqiV$lov0k%N1n28{z5pxQ4?9Z0E1IEp*E4b6uI)%rS465zP zT5N%@g=UB04AP~Eq(mI*T<#=ge$7+s&_qiCfiS?){KO34kj|!FkPs-Ct7D4fw|fgg zs8P++V+gxdy8&sB!!D2w9FEe$rabDKq69X zd?meODMoM#iq%;{P{i2vq&p<|4@z;gWuHkKC2MXgkwRbPYO5?uo(e-{PK28hnmT}( z+1)}AWSHs9%tTc2DTO0PeTPz|0PrJ(qoL|y+4O>+llhgTXI>AK!a;>DwVc|PsA@vR zr6=Ppkf=TO@h-CTIYy`J=of;Ln>Ej>JBy3 z)Bu?g0_Y_wtr7BByU<|AL?ezIjt0$;k%EHlWLu&rx?o{tOHI5I)9a{jFDO~o5s+Ox z;B6+EBZ@e zlpdvO?{toYJgn}oY~vuCx@yTaVk6DVYp|`MQL_`UlegjrL*X$q?shd4L$IBOonxIk zn~8KMi0})$02`NxpoJ=lI_mJ3GzsxN$c_c}PgIcO?NIi14+={}nH!L`xD;b<0o%7! zcSu3FeT^So;(!sGOapSuR_98{6_|3d{k@6>ic7il1P*GnY!33^M7R zsM5%CH$iyWAxxNc=Xk2s_4%6O7=go1md0Mz%zq^|@ z)5C)erVW^=U-dXIgZ+9w+9Qi`Si|*t-r98^hr>KA?6CH*qUb!)kosdfCN&WptM%w= z^-3h6ZSlfXd~;&^9kypsib{j)feMt zx*ijNHxZY9JM8!4R#dWs<2D@$yT4j}InKLV(bG;pkB86p0yy7~TX8zy+j6o&DV=-v zP6J{iv;0}y&3k8bUdCnq1evNH;O%a-A>ST0PnR|zHd?lj(%3u$cNg|$@wpD$d3z2n z{rcYdrERTE3)nUd*t@%VwRYa!o7HxFw-7gLE*=tudbX#w1*m1_H@gs5?EUq)OVg2N z$nn6Z6|6_I5wt~{Aj%Je?Wwii!v=dJ{$3l{>*IcJ)BK0s^rMZu?K!`YgW}g(l#k=) zYW2fHx@=dJJ-ED{ciWd|zj(&l|8e*JmmBf^u5EVbhiRLD_CK%QO#7d0u1-Y9e}4OR z++GjsyNlJIruBN>y#K##wcHNdX*;DI@U`vZ)erWzkJEPlqdnj`a`(&*IN!Cs$le}n zdTXz(bv)ZTp01DD&;mRk?h+q$!r`WYUGU?RY}4C~P3L-iL2;**{c6};+EA)_Ac4vE zho{YW#)JJb?rv;^YCfFE%rpD<0dcQ(V|oUQf{Oj`1*g`(oo!GjWfm+vUHeYR?q3JV zZDO#{7S$6};3okp;re?_EQ+Wh7Z@!P4ON>}d)=#6e>RCjML8vqSTqNbI39!dFk6+bB`_f%Dz+f$-+hR!6EY{uC$(os!NedKwiiq01FvO2d{1F4192z z(mxhsVW+)Hls(n*ZHx9ft5$oZKFyX1NJ<^|EkYI~C>$Xu?pj-vD1;>5g_>|iN<;$m zF**nq5mt)IGIg`GfgALWB>{!HV6hjEN`8+ZJ0D<}4%$(Gu*;Go2x}&^_5O^IS zkg7(^<}kLBwIS75v!i8J5-7aS)Qh&TL8=0a#mW*QGFO1OpTBItu)XgyT9#yB#e0rrHkIsPhqfwxQXS6>98(H}&XvlL#Y(NZ z=+)FnCZrr8GrvgFo81cACPa>)!Vw9LiU}-~2nAD06%O3Sj=S#TRh24f>Nclba!_vv zpp3n*g?cz_g%b0kC|sJJlI2Ysaw5g0d|XjxOL($~XzA$9U!>xy{+Nnn6;;K{}I?|I!;v3hG?OE2&vegi%dTfqwUm+a5c19?ojINc} z1OG!Po=lF=O`l-PLW&cp>tbk()pJ(3D#qY4y4j?^}U)Rh)g^#B^P zGosIte2a5ZGnBikvbvPNxtvad8Vmf^l51jHy(W}=olu%E>cT5fB}>RVWC0VWS5_m? z!$~s1c$y=$L1wxLU56qRb+))O|gZn71r8qA9h)B-d$f6sY#%(Z`JTa!JHm8Am?D+h*FBsIIX$}s4k}y zpF6s$#^#0ilN$4+sU4`PEw+eM?r(%5!R}0H98fLX7Np2($&c>6zZtnELDt-Xnm`F@to&Dbe)}8bMViv8ZHBpO^ z7rgb@sN0j8L|k719WD_>WdytaTy5w90Uex%3h=0M+j0VOH(#aDa_DtP{hGXT3v3z+ z3C!K+bm-y7q za9BX*Op3oAR!I|cFjtCayROXoh>HY4w#`coXLic81rj}V0wL3>?1P(e)Fx|gHi#N} zY>fo!wG-4=dO_GcCstoN&7^OZiGHVrVoRF_#tv+XJjC%8VCyJlsXlmUKJJCiAlPhd zyl)5+1Er>zC%y~hya{J3hBL6)l8ga2+oFZCMshjjV_I$y5x*8xqU}@V5|BF3B1@7p z))~`h-9J9MAWBs$(uy+ItnJt0ig;!{%WY9~S@Ep(Bgvl_YU zLX;GX3N@SmA-Nr$+CRzb@x&;!dvtY?buH^-b?9N+x!m16acHM^8FQSzJ1{O#1&6EH zCCEG}USKe~CZuAOlt;RZ$;&nGBvXlz{U#I=t?1Vknv5D zDmj9)aME*&3h5~6WesS{_3}g^n_D&RA_IuBRc9Y@>YGyoDa}p||JVpu^o=u=ATgzd z^%+(uO?^;$B!4VB6p`eiw)3Y#T6YVKB*RGD(`Ifc7uSfw8#3GqvND6_wjtTI)T;5Dp7#h(#dYSb36t572pe2)VfA5~uY_DNsmck4t=*rBeG8jx~)G zTBwCa-SyNhBH(C+)yP&FJE$Ti(n-?NvOU#MuYMy^Dl%rJGK6?0bjtWKBad|2=%}h3 zR!`oRIA)N$NtT?F!aCd#1}o(d@;6%Bo|#&ZpWbbOUH=L@I^xQ)qo^%2(iV>g8cDz+ zi?{jY)K(*pjfx zwFF}zZD9MYilp=>wlYd`s~TBn3d~=-swnL8w%S={5VlFDG zlt?UXCw{D%(?SgxJ%BfZsBW^K!8rIR&BThr7i$QC3VO-&ejqD)dE zEfb9{=JqU3vm=8@9MqpsqI6#;AqA>~tH-|2+BkiPl|zK;wS}VMiGs~2maz&pql)lM zZEV4IgDqu^EK5g1I_AnVili&O`k4PhCsSxCE!3!EuZm}%|KkAN{~4ia;(Qg}&Oc1# ztk?5#w|oDK-G27}9@fGP4%_K|Az5{-mkOo8KkeAe_QS5qh0gvxVw3}ywyK{_l(XQ z-oF23VS^P3?-|$Z)fuIyP>6?Nqks3|w0}u3&411f4wa#y`F3wZ*4h0rFVUS{W`Anahf-i!+C!=y#M9lZrF~iFL%SkgDu7B3~Blwj??zw zkAHJqeKj9euWcR#-A+ue&pXhZhh+!a>35o(ukY=CqJb7d-f5`yU9llM(XV5szZ`e> z_Ai$=u)f6}{s@L1{d?=?5y&&S&_{r6v|@%Yo_>v=QZUr*z* zn4WRhHiNU+x8IDbzlbT>Ven7(6lQB~jODRx;hh5p(Swh}HOvLWODIbrQ3K0xd+AwHcpb%CV_9i&3HYo3hQ8 zDi8u*kR077RB!8A#9$=TB^6U5&8k(lq<}^#M-;0TCbB?CM6Ygo*n*OOZ=jAZ5s{j3 zpLKL1N#x115T1x_?OCxmUIsDi|8xUh2~|fsvu!glkz2k>UE{C{g)GP@WYf7~OB9q< zi&Cs>*zw$yf|o;6gMD8Zh7bP`uf&_wvY#vJ690w@oH0 z>(3ja*H>VxxsVkewoQXt!-Cv5S-Yn1sa~srOj)lqI|;qnZtF8hs4^#Z98h^%3_Te@ z*EdeXMptn0f*76~WeV|(GK+|iiblCCSfk=uirDcQI@vyC=WCU~rPM`G_KR>E!7Orv z?^yMX7jbA*s*D)K4Qbcp0O*wZGFy{LNv5BKyfo^^YEuc%{PCh5&5tBHWN}_ z(zJa`Lm$PQNAiA6;i_MhbuGOypANPbiJ?6)gQX8O%mW;&6T?x3ETtlA$(XO*ZIHSk zR9BqC3s(d*M~ zno2a{h(3=h_L&EX2p|455#R4Jm(}R26ixmZ^ zL{&Bq8Czc3YN7jtns9IDNI`S!;6v2&6rv$^+=tTDfz-T3wb~ODSv+Am zsM+r#Ka+@cylb$6)g|)kB12GJCMO?mPzWo;N2}c$yM+Yv#zu)~D}oqim5U(+XVW3o zaCr2{o&AgS^yYQDoCE@5Pa;?MMwePp}XV#cGEJaE=-;$zio zSV%~z6L~YiS3^xQJ8GJ(5RM};YE*<$Z%h+OEvc_KEhi19COZz6Gbe**`AnHdob}DR zM^QHw_3Q?5h?kK6vV~K(5HQ#F<{{H>@n;hG90~V!eP3gbs8ADT$0)(*2#-myC+?CFFlNy1CQJBmfMraYN*SN43&hyVL1+#_%o9E$wSOQNhufKD-MU_#6m}M zDWxYDGpgaVwag1G5a{T_(3Axg8$otpCyOf>6@&}DO;yImnXHtZS`=#wyl2M^41Ogd zS!y%yq^d+tWnUsaa%-EKN(3^3T~=K4U2SW+pt6a*LtSMA)P*We!8T<$R4c6^jaOwsG)ednRKaGqQI<#PbhGa;z35hl73=m4OVw7Y7r*{0rkw6 z>98lR)iNjR-MP;~?W3X~M=b{NBlW}z5oA<*+W0bs6(O1B6Sow^H|oYG%(=x>HD#cp zz|zQ_6s2V2o4Do<0BlyDg;@)|-Rv?nzu*AkTj&#P{_ zHfsY&nJ($8K$N0@=M%kb^3yY-5MPdcRrdur0KsX+u@Rtc`aaVtOg1b8)L^nV@2MqG z?nh1XQ?)sY9sIz84SD3ygk`4+C4w7CZ%~b;r6KGD+edkvwxZ&oax?Y|0BEbJo2BoDXmpBnXa8X6oF`)UlT!yQRdO#63G= z+is&M;vkTYt(kr(OH@L`Sh#xj`9BW3{r?1e_N{PmCy1&8H`kkaZ+E|b^3TR$w|QC# zP$LxS{+a#rFzgTGezm`w-a0OIemCrHR=auqqu*vXZ#E0G@P`H9nLwG_(e8IYZ0F

tFiQ_|8UPx3#M~3h;;HcCT@JGdRQO=-9*WY&H^N^3{C&m*aR4!17P??&gKL z^eGL~TYI_baoE_54L|yGug?*pAEFx*9`+kO*&mnRJ);wS98XtUeXsu-w{|xhzC_jg z=Y_8IBW~VvK(P3E`=-5GeKD>_XL7akzxJ;=4zv&)cGzwwdxib5dxqCuPdodA{l_0} z1V8<58s7f5|MI`Sn{I>+-Ho?4&w@$s7hLIQ`xkQK_1&;NjK}@xOXb}tkgyQPdgX=l z_W4J*&PKZz9@bXz^Cj#Bb{`^!?;In0J>9PEriYcy%4)wJ(>P4adi)r5e)GaGE$aP! z8Z$BDIuols#;~&Q-Hxlz--^#U!LeW4?XIuK!MSJek6W#+yK(a{J|mkKkg;IA7pvRJ z@xBl9W;*zmaOm`G%h})Ja7_HVj&!}Bwzf$R(|CHd*EW;)gZ}L=zE&S6EtH$1P3ewph`@ETt_VJtJ{q+9dv_p3D-^cX{AfBMl_p^p{^)Rgu#|7a% ztRB)%GV?L{P}ttlFTUKR0kFMg6Sa!n%c8LpuQoHW;II90Z1&c({d#x!cV`%Vn{T_# z!LGDl;LA3%vu#nm>GBG`F$FaKc*d^B+i9z9_yJ7$>iK>?TVgc!tJ(gT?`=|UhNsm_ zZ$WyA@5i6?UxW6BJmYO1e^m%%xUD%Nuo^92ErrLO{jEaCS0EE(Xb zsb2mcXU1$?sOn~JbV-UxqAx~0vk+YjksSY0y-U=dFR~SAiZeIX_9Eg|>^&MpMFnYA z5Mxki&0{4E9#!^{)z5;x?h`yH!5)kjW6>%PpFstRgj?)Ulvk9k6`{NvbEK)7EJ!G- zt2BMfd810*{-ii=PIg~qbaB3Qz4wmt4Yf2(h;FQOunPfV#bnu__MN~R0KaLa*+Dw! zHd!?im_`LrCE;G>Y2zj>>n@|nO0^l=)FgkZqy_OpNC8PwW2lXmwV`Uk3${Zk_@I7P zG6E8+#rA~8+uG?ZlqdZ-T8LLsx$Z9KJN6jL)-}C^Xv9$+(i6u$z}H%lAR;qtsos68 zgIbJDT)oo&3Pgt|re#<<7Qi$;mbNTJjOvkx_mBilK(=yE3jlB=RFUFPX?yoryljFQ zQq;n8LiUe~bZ$-52taUnVWS(oSlf41&J5Oz#jR}Htj3;dA390sM1~7oQVqBh-94(x3&DjccvXN~VneE=8-@VWroqmex)RR;oQGoT=ImS) z*?mSgSh!HBnNonX39gjkR(!M?Q3{aZP<0g2!s=1zu@SwHh7$aukB;riC*9mXOS} zn=FUq%Ry-`S|)w8U4neXwn~wusI*go0J?(h)hydyDJKg_Sv7Ce^oD{MMLw)L<-Wl$ z4Q7~XxRgp~QnQDEK)_`v&oyEZr|L}|WzDHsrG>DzVh0OqIerS1uup8~lAQunnlqYr5@jl|!H>qCOe zc;Z?hPy?2tQz;Jy*TR`PPu%-3spk;pH8wUL1#}3A{G+Z3gJ6c*h6GQL!ujlxOhp`R z)cex)4bmXhqDe;$g$T8b#ThOmqMKR)ASm3BL)GRoC;q2Kf{J7S%?qlrp{;k>klgrs z(@}{u&9)5{QlZ%r93KTQY5@xfkZ+DtOF)xW1X0AWW1C80>_`_uE&Vi`sS6s06W-#0QG3D<@e%3!cbmrsx;Aacc*Gk|XI#`?9SNf2KdHQbe|_+rDa1u%Us%IRyxMQf-d)|Mj@$BK`iATyE=vgd^WN-azU z)dl5Lm4Q-Dp^G)L4w-kVtSzmsUQD|%^MyL9VWEnX4LwJB{|L6Tx$5%dXpus`NQ*1j z$~IRQy=gm_Gk*Qcb5myrDHSi&!o>`3Hc})ss5CV_RIe^LKMa!Q~^S2SF*xV#{$unYOMf@~V*^{KD0=A0JqyhQ;}d$qD+*84GZ#ciV99*gh5Dq%J!cUi-d*o`qqbftf!2iJ z6yHMov1*@2(ZbaJ6Nhm}*@s%qqcd}0ZmY)mmuTNIr%y5xA+OlT=jCc82~#)!b%aM1x)?fuu!FXQX<6O zHcHjgDhjn|exdExEVd|O^Go$b#?U2TNAruAaoSB{Ji1VqAeODl@f=Y!2 z?c9Q)jnQmYiG4A#7Ktzrhl^C_3}ns`}cZtAS;;^wZ!g$@yKktMRAd-j>YO>JP`={4g$f)6bp6RVa5eqMST`$3|gT^MgKRxfZ`U#yAz)N!H0 zw(dT<>fLa(D=gQ(u&>QGtNYPOdF^NRi1t_y_SiS+DR+mr$4&ZluSvYQT7ACS9Dfqm z?#rbeS^XE=FT3ry`h0u4876^+ZA`WcV!7>_6P|O}tq0++pJ{*f?aS}yCtGZ`hCbgO zY;le@Q^WCb+H9tAxBtDp${EA)Y-8y!`1a@+*fauXZ<>I?FCPd4o?xVZegDhb`M5hh z%Lg~V5pX&2AU|)HhjBN}H|cuI$Yysty-DcLr{!bLTVZfdpzaf??fP!q>BHBWkSqu}|wZed&9V@m?f^u9$CaqC|VGZ70%R|Klsy3tYf_Xo|#qIqrf81)hp1%(kj?tC1+ z9e$ebKl2Cq{;1b~@^^kS-po(q_B169MBhFS>gzk(WBbeZ|GIhF+W_oew(egvk2jxv zZst$C;OW=@;W$hh(aqq0-|AVG{q<|WzDg|Q>~1NRs#KYwfK{p**u)e>z6QQ+ZzpwD znRdS~v_&Q;rgV!URqc6s)HnL-F7t zC56Q-Y9XcUxcJQw{$nn6m9;l}>ZFf|o0D1#@rbG8yTDYMa^qblN>X?Nvt-=Lb`2Mu&G53yvJJXu zA8?UGrPqj$sP~*I8pa@O0VdeQi%3`+(ZJ)yBr+W{j?Q8!3R6?M#N&}}zUo!OCR%<8 zS(6EfwsmNFRZS#{IDsZIGV7EtQzf2Gk=~3l627&evil$>xcFj_PSBoK;$<{gvByed zW2UU^D5)%QV3F*?=51PRv5DgT&XuyNE87ntB4$MbjWt%o#njWm2DU-02{746Lkcw- z#Yov^EELW>*)r_E6@rwGP_yAccF@^gZwUV3ZDTZot>pdCSl5-@Hh3f(cJBA@%{C%w zg4y1t%{UI-3{`4BA*`FY6sb}z-T-X+sVY%)Qqr#%Gy~g&1V=(j^u~+;ktBOh7l?Y* zMZ`rd9q^E`rD#mLt}juDyRoef2aHnI89lRqZr(u$2TvuJ1!|?SX}VnM%WaHE=xNJV z(o3h}l%u&2PS%LB6#2!)r4Tc$SK;susVX8)UJfJR#>{buAyeE=2GdS^(pGSe0xCI6 z*c4L9VydOmqk2WK@>ttgkxD}&yeWe6)Qm`LM_7BL;g#G1^ikBYD_SBzknX#^AV7VU zKStDvHb)G|TOyW9#2T3cWRp&vdh&;{ep?}{Zd*#|mHY+~4PzJ}cAuuCf_mtMup@v& zUpa|3443$&z^ym)+R|cBg{?&CYO}dr$7((zXzwYGEd`&?+&q7-R;WsRbyRFmCOd}{|jVyetK5x-D$a$)k;MN~yDuG+qlKvIpt9n~0B9HI)j zxp=vpP>K|Ur@)pqi)_GQ?1P}eC~we<7Kdneg)tyG&_)PyvNq24Mg0-cCMrXLUMBvc zhO$sWJjXS0`&jtbwy>#NEi7to7W$ye)V$pcMK!>ZJu6GWmC4l*u@RibwOX(s+fs@z zAB)67xmMWkE+W^4@F-pEK5-Z-n@5s2au+Q?fywP8S_Dq7*}n#vOryHDBS03UAk{$5 zOkXPiN^3zb5Ro3HD{}wn2A`XG>p-ybl5ITvH%mxy&_CZfYvAyl-HY0pXXSa@f zvM9lPAod*kF4I2og<`%@0LZ-9Q$qxj7pZcGn$dOMhmI((P~{yN$#EXm#&NUwB+G zbrVrfCmbfV67*($8QI;mBH)L+S7J6Nl$9O2T}D^q2c;UBDIeTN<%6kJx>Z#qH<;$c zcbLAYgzlpxwd^0_3ZvK+iCi|-aS%4G&3XruYoW9h{z|Su-yvvU_S%kD zf}-nh#1CMev=WTSjay?!Na#F%!bS&gXTZ}^4irTfGT7MtCZub`?7cl0o(pxWID+c# zLHXSbR-at4E@1IM4FxElQH6}mw(!<0NjZ^DtDt~z__S*9rc#GaD=aaIR4L0D=t^z% ziJ-MBFuhr;-4cb{Z2Uw601i@Y`%IBErApO>gcPEc(s|evt%@7u^K89UnS`pl8-hSq z;4!O*y2lPa5t^47b$~C$&OA!kaAiv3(7DjZvS2M93;tuKpOZoo!96ID5Zw~MImMOM zWJNsRRTc+kReYa!^I5Uq()MkLSy$&}b)`fNR5Sb~?d-8LfYK5{@D$+QZOq{+AeLyP zKuX?IDN-_YI?BEhm=xL^WIAgoC##PmQckn!y+)9qmUUSYF}<0Sy3b_ImLRGsvgT($ z6>aFV3eOTOspJaFt_e+CFQh38htO^2Z#?(A#*g;|Ll%lNI3ejB^!m&fU5d_j8-;w0S))l0k1>FW?)7T* z^;Yb&-Ngy57u%O7^=n<~TKww^L6jXKd%`T9z=i(G`|+95_j*2j_&V?I>?W)IxO)`4 z@vu+a#V1tp3EO%C2fz93^ZV=FY@a?)%Xd!UD*)tQ=l$&qs!#yr^NVli?d4%N+)T^t z-jCbk{(~#;56jD>pRMQnYmp(J9_9rQygb&2NXuv1Twh8Hz|_X??FY2N-%aDrhUv@U zb~{WvyVvKN!CvHxKU$Y|?cY`GfGWvM!qv(4*ndOVxcX9C#v93-nJT^Om)LCwz`*(Y9T zN4V;byYT~(XdAf?iF((^ja|V0V>7*7IF%Dr_PgD@7W4Ch6MZwjJ*-|0+aEtf{n~S$ z5WioiFUI2hd2_1j=L?z1*gpu-Ex7bE+;=mqzMl{C_GaPt9amqx|9|dJ4`pNW?Yuoa z47;sna`nR2Yv2FlMn)n1YQOqoJpQ!$cDS7n4xoKO@%~`*d{})s?(XehY(zJrDW2?~ zlNuL%$?X@2@O1w$?+(I4-;d+jyg8QoVQ|J~eRntMU;Ne&chhFx&kwc$_JV$2PqyG^ zvGWeA!_$M4E}yRf+sn3x)E6*{6x<1=ht+m8Gjly3cXxuDpW(6{VV@A)w*GI%wb+bx z+ud{`=N@Ez9hAQoQu#1$7V|5z;@fd6UgqIo`$x|>{z^_?6|$+PJ4zjyEs7X;UIkYgADW(RAYef_jO#7%MErNmaYf>VxjnOV^p2%G9=) zA*2e24=EFmb!m$N$Vm=V1hjWhyia}XO_yWWmsF+Xa9|8e5eP7%w6*n1Lt;*K2cb;c z!&A8%T6u3^{)5iq6#{LDH8SfY^i+ebkcbGzN=g*ac50tV_YDbO`YMyfyezMJG0655 z--+|j!eO{cuY4$AK*aK30RSaj>kQlH@k*I7SoxL>!bV|3guO+Ju#G}xoea{RVlulD zRUTx>x4uMxoziFl zwU@-+r3M7b^mJ7&!sC`-aAG>v#fE}(MWtBKVUrYs+Bb|Pu2jhZRD(`#3&>%u1Qyg* z8M`aVl1l0?6Z%d?wMJ}-;0QRS19Z?h+8qmrb`~P8pe=#a($A1q6i%2gtD26Ly?U<= zz?j57RHURN;>?UxCYksu_8ET57k$T|5kaC$YJInZ${rY|UCqP+1&bRMgt+sg$ky+Y zm11=?nUlUJ75Bx~&A`%I}b7Q zV}sZcDpBRkmbfP60oi2H6x!rg8LNYAV;35P!IxsBMzEFmRk8e9N-P;mcToy=1RD#| z*?_}Gl*MJ1o7XSO%Fb6ftyC?@D$oA7!i_=Up>-JsFhA;#hf`?)66^N($sb?p8|e;tf~x7mMSQs z5&~PTSrQF75w!+^i(pF;?@=AZTL{<$HT89r-B+Ho$SMeFt*g8Mr)Q@ou8dCH-!Hay z6cZ?=sihl(X16V%-N%Mcz%<%7P9TD1lByNZn2wM-#EdFJ&!tW$5=dygmkK2c{IT#R zg*qfrvC>X3X7F&?4}>SQIo2TA-b!s`Xsyi=f@V^=PFOiBRU<(HC-t$S_O~tHDr2iJ z5_2FSM}>6lNVhQ6FED)}kC^xx+t6e3BIxxp5^G;{t?1||P>?G{UX}&QP3E<@P1X`F z$YU#dw_98RR8I!|MNOL}=?7P??gS&>Sg=6epUa^G*RI~Z!NRibk-&U1D*u7^S4cF5 zpfZXR1SO!pzGtgF+3S{Sv39b#Y7U%8^0DX-f=FgG?`aGz{v;4b6?L=&<3Xne z`^+wdpcDd~Y!+&?4VaA10}67`MLR&4G&^nuivV9OScr)|`9jiSV~ZL5uFN`dme^Nu z2>5G_RH!oVvb&0nHzr|i-W-so#dzK3N&X`3^tNpgJ<~b)JqF)>B)&T}$`wd6CTfL9 zFmZWP7R^%rrnPtqX;KBStB5wyW=pJPu!_V9K#@fgUAr4hrj`*VF*~`kl)Vy=x}s|} zqWPfavql?^I4;#v8Q?&SqWD5~16!@1;^I?dD*eRlU)iBZwe-PZFM5WEqA>fqMHaAx z#kr4E8#aM4XYFo;ZL#r1;`b25>`&*UOd!x6_u^=PE~KXKb5K#5K_~De%jguw@C)+` z&CjBRrEW4@*K&)oZA+05;yZ-P%^D6<#N0H3wm@{K{wRvL8Yy%k;6_a($l@FPk9@F~ zneub8*=iD})7Z|dimYHGMXfAiE>e-BCYlC|m0QWLk<(Zc?Mzn*&GzxZ-du@4#=-{G$ zw`Lu&C6tS|jm6onTA~?*Htk)hj~p}! zJV6mu;v$ipq`DFY7xEZNJ5!b~sMCw`t`Y~J-fat}W7eacoT|#JBe_$oqZPnsl^D^p zSZJ}OCHBVJ2_D;){K=-Y%?_mo3%+epAx5@*{DE)y;mTJF>Nc_ zZoqrm28}I>6N!>O>kKJUKTd;)Jc04CI2+WFm>r`jq?Rld@cM}Bw!lfLy}g|HK5QgQ zYT+?wp6r9Yg{pG!xOON;a`nVI7OR)y5e(bt)9O|riVAE`gKQP!3e zgUvt$vO~3P-I(>!xDgbMF|9eNAkcb9)udV|;QpL57P9==hTLJRP0S^|bF+L?|v0&4A=*3IH=2?Fr zGPWY;p-M|$*bZ2DL4f=Sn?d|Ywz?rfDTTHca2KUS2cS7Z_^Rf=kHz=@58$-#N0FTR zVdz9&49AIdO#E939RpB&5B;dfp%G!}WasFdn7^Qr4|*rnhhH+V;H5 ztKJRQQz9jOMp53_+b;OMoq&eB$;M_k-A-FaE57q_I5>an8G-0e2i<OVc&GbXs~8(USQc#FTB9);6h zpvbT9Yz7Y6F>i+TZk~R+S|(+gz12UDl-yC1&;PtTEX+$=* zg;#Ip(c@-1xd^C%zYhn15R+k)I4o(X(EwJ0{D`T(w0 z-`cp%k9y$+c)d>))cWn+*(9ELzE6{>RtOP6@7>1OwEWZR&1Zjh!tL`3ODzKDk7z@9 zA`8DCeiU-p{9;d@WR_v>Bckf#&!Jl`UtzU=Z)fty5 zD=coX_yy>|?8HsRIoDF@Yl!|_vglmUzmg1P?L-FkFthIcbto!WO76gCNTzZ z9z~EteuueG5yfc442#?tk~$6?{qz!@p|q;PTL2q~;|!r$X7h_OB1eOHp*3pR2}fI@ zh<95F_wvzC>MLR-GsHG{G$4i%1nwO1>{P@kQ$lSiV%isBaiN=InL$}5l?B| zWPVawr)rnSu3Jg}Ay`pG=`304zYtv%wyk@+H7IX2sx}7bMic!>ONnz~ez58;5^Tn8 zwDvcoAgU!0kFj!Pk$n&4zAA>Ew8*q~Ym$_cVg+(h3qxJd#*ze3i!|RB)>kFC$Exp? zh@Yw?9*GFF;6&pTMOv6a?T@9@kWn2}0c@r^R(Nr{Bp#khu~&scf~h$jmR#(`M2!TS zVyWhWMQl_q&+gx;nGnjOO_b`%|4ZJxEJ<=?XPU2K(jpuH(h}vvTsC}fxinY}2Ag?KsL@IrYO|qn<%6rY07A9{8+iYEt()Q7{osODj zqo?LES0>R{2JFs37apAY+o+iHK{m zu`MlLI}9W%Q6TyRu|@=NLnIM900=`GdCf6va4$wTiy2NgQF!A`h_N zV$@5LP;3-t2MN~4asky&S)2+0vZp$mh85d7&`)&JByxAg*Vx{Lf=eqR))B*2+fQmh z0C!xb7u*&z{F2FrUx>)50Ip_#yoWxO{EZjA$j*!U@)meH^?ifi|aTniw_8 zkQ6ceiruqZZsuk*Me#aUbb=T}ApBF0N<%;CZEQyrEU?(2D*~|E!=}1m?(#!t)ZmgE zrzS5{N35d?3si|P>tiF2-}Dv*kvG+gp#(!l`B~T!t9nO4keEXkEl1SEF$c52ulsXJ zy1W7Bvz1%~H5xQhy-!4*WxkvRb*L;ZsYUFQ3FczL$_~++oyhiDoJ+ZtD3lqoKAXR0 zt_v~tjCG0b&iRN|Mi$xf%=`s%^voR>-8eMtDzJcHWR#2#Je9HwP0#nRXua$n@))}p zqd5Tt7FDI*FId!2>)g>9 zd!%HxF&9`IY??h5U_UG@7Wj|bSq1a^c!PFVwnFv#7@3>bWFaMKCnVRmhF_#s6s}-7 z#I%OF)S-6lEcmNZ;f20^XA-5&qZcV~W_e{dkU;luqNH@-{2Rn*M*fC5_dNwYU8ib4 zGIlme<3fRornlHXT6to-9g)&pDjw84->cqFg0?~%Ac2L<83y1svjZAOQKL5q0}v>N zfCHT{?j-W1FjyEZ;sG%WQ8LEW-eA)-1-_=vIZJ9_7x~-3@_FWT5ZQT5>L3TBEjlyI zI-5f|#KK5u6opMgbec^p^YjniaZkbKAbmiBPKCY~yFaWtIg|yu-@_jBh>C1qUMWg6 zB6+TKL@3KTSU`fv;YNpV(H3)=Z{y4*F!mzT#Lih+bn-2Fl@CZ;rB?<3k}{oSS14*$ z97v2NqmwiuTdA$B61=LPD3T|Za(fN)UI{`J2PMPEbsPk$eS!K^lojQg_r?`++(m^> z>M;V+k<}FPn_>@(%}2r&HHz4=b5@s%J(0c+!mQE5ny1@C)v?o7FSel%hzbNrVSVmKObc~U*h~7&pnM}_rPK>hBy6&= zYuVd03)mIOz&aD_#v&TchfkBRTqxxZ3$P%cDr#4DxtbxRfc#PYm4#F+DQ_H0&AjFdFhqw3psj$$o$kB456s%@?*h!=9{3g2gahc2)JNp2A5~!OSw!J zJeD2QP01LTH-Q;TO7hyKX}dzJFP~#g2@uLYc2DYExt>D|`woam{j<;iayak*Phf1X zMC`iU=6?FaZeOAp&*ZH0dS4yxc5nB)-MwB`fI3X+2?r!(f!`(am=g$r!6spjk=ezal{jU#q zLR)^fwtubW>S4dTIj!b=o!9HVy~2YCMz?|)KRjMbh{VYWSPysh#}D({&B2kV`}OrnPhQ5&diOI(JXv^=oezQKlyLbk*{l(YmEBnJArwm1X zHJyY|uYR7cr~S=U_1)=y79RE4!#KU`>TW#?E1lt}4g$`L@cF2nVdT&8ykDJe-~V}Q z3-fq8A1|uyPGbO2K(D`|pNl{DW`0<&9M7uX9*(xnj@7~5P6+BBch~d&Sp9CdK5Xad z^zm

AZV=+Elv2X`%Xkn0F5gs{Vf3|MGaT>eCFV&FkRXxw@X#2U}Mn-Tmb5{eL?O zemhrh)|<_lFMB1kv9`%uug+U?zq3_s*24bou)xy&bw9h%8d%vizSzCd+kg0=+x6|; z(_1|=D-Q5IJi?OyZd(2EG#@VicHZvS)wePVwub#>@<)rExcp)-yRx-4Vyjrtb{i!( zcV}$;_8EJxzVsFTF}bquysd5fqgjU!d5ayE{gre z-UMIXz5jKu2|myF?;Q(oAAEkZ+t?zi{y~F#JlbwtD5}5MZMV)h{rh<*bL>3E^GTf5 ze!I)xPuFHVH`O=S^X4Ga?g;HS(|)^NXsd1UZ4YMcYy_vpdcB%&)|3A6TQI@Bg;s@H zRE`sryu=iDuO3-)sG7PIHpos&jZ8C1N=`((??h@?Y_hnnBP79K?k9571XW=mY!A{Z zB_#rpU5&)}If%T!M$!yvhoQuKFG+fEmRm(T;A*Ls9| zI*Np~B}p~AcfcZ>Im>-Vg#wFAm7OHGg!0=^!O#niSnN@fXrQ(+(J2(^;+#2aF<+p7 zITjIfj-{Krl%w}`qWmYMgXFQ6jnO{vT8kvU)HSida{Buz!;JdsD!LX(hP(~C#Ya&!{g?_gi|G7kaPc54GR21r+rV)6hOorU+I$3mM@ zgyn9?sb`@)urIt(Dv&A#W>or~U^>dh^<=bbt@?(WdPl)-==BxwGVrDm;R4Y+2PqU0 zGAv4o7y1c=+ZqeL9k)fQY!gwb=8HA4#GhK|y5%B9myj$_>5@bQ@2ZM+#KaI?xek%R zqO1d3uL5s`H#7?w+CuX&ua*=_^`2aIXltxT^hrA2xV2l=+7NSYryQB3azt*+gMUK%&ZY{7F2HTFfPQMqF#Qu*>?1kCmP70}Ax? zf+G@sGTGK?kww|IwJ5|8j4A9h)(llLfP9hbC1M_MOj3=4;Zi=C>EheIMn%0SWl~bR zA4h2#(~`Gs7^uG_*hr>GG3Z%GALv=4nW`;5&!wZIlcM@us@#8gu~So!Fd)_bc{l-kdd zSf3%p7)AA6kns504Qk4R*%0Lnshp}SqA*#0>`J@ zwsT63^=wE{6HpifH3C4upqc}6f{2Zu2%%!Y^tzZrM)ug1cSqd(C_EV@-p&83iT**V zk|jLLkDb^%ohBWiaqP)nikDN0kg$FfASV=1Pl|Zq$sF3=2bdFaW1>K^fXM-Q&=l3H z#|2L-k#f*RiR#TLfbzat#3%&6NVZK2N^|!`l6slpxso`L;G7;apD`JKJ4uSJ$2#MX z$6|43Cr}M`5Li3o7o8VMfCRG2L9sY43W;gd#Q~RN!sH+cSV;L|6nlyHi=pyrN`&ob zwk{S?WUe8P=A+@36auoL4s-JZb@(6~Wh(-6vUr0HGPRywrr`w+p!$fmaluB|@yOn4wTJsxx=dNOfn&_al}v zDEh$&x3O(H6u)tDzNnT$#HHHCAEDZ5v)dJ*U!M|-6p<}5Mn$+-8kQ#-Ao5gKJjOD; zU^Cd3*$VDQxbbGJC7Du=ygcnfuOLSU!smd`HwvIq<|XZbh_)s$Z6;A-a;;)^EXj6? z@g4Pp%nt<=Mo?Qq(Gst*8i4!P&5C(T*idLO`bgwR;+QHB>;40&tQHz!OH!jzy__~)Sf6PxsM0s@KFJ#KJ6NL)kwsj(L;eX&n`pF9DM2t7z+Uc_jpBmAP=z9Y0IfRGeD3uwYpG6b}Jt8D}d(HyipX0|to+{OMEX;^VAEU`7o;~kz z0yL_*=CX6pt!_@863EO}D{6UcX-PfOFqktC3rf4%iKgR#pr=OZ6j-Ec-0QKeSUzx5 z0Kt5>shC1RuboB#G&6&m+({`yOq5*@PNnN&A_sp^{~DEEhLmVR1{c@DhRNV6M6?SO zb~3PQlmsV=y2V7vsx+pt;NZ#OPh*EYDgej}rGkZ+9Xs-X<+cbDTUznXo4a6m(^ykl zMm?Sbw#fB$fw{~Hp4KIp;J}rz7%UQ7=n2&>2GIb+G0`Z3P0*d;4Kt*ZWu|8+SVDYg zl|rIqW2>Av{6c)Y5%yUlhdYvvUe`GVGE7pG6(!3M)e}iX3xbW}F(C>Odt_v-vG}y( zBDQ%t=(u!>Zc46>_DXJsok!T-G>F=^z*FbdIH;2yZe&qMPnw!i06;S!Y85*dX!F|5 zDQB(Fozx@8A;|M3<7?+CwZEvpsGy+naa5s`dg?l{{1ekyl(Z^&f9;@3D|$e}a8h!_ zT<(?_Gg2!uYfPaPz}q20<{^sEj68}_U9Y6X*ebDzKnVFq zWXYRWEKx96LM*ckkewotDlV;(t<-kfs9X&0c#sqZHE!EV?dZxoXm0uoEaD$b9tUotO=^7a6}Gs5P;%cXs&N5WrgwjZ`Gy zIFovv?XLeZnDBquW(1tTW1_uXZFYybI-IUg`|C`xdnPd5o?(#N`R(ChdT{>G`{{Q2 z$MtrB%|4v2H|vFs^BLapTL<6X2%b7uU$1Yc{m%ZmzJCAfe#%6=_W5_a)0=ryeSa-t z*y%W*_WtpglK^p*NJgjk|NHj+ulDIweJOHL|31S!=WX@rdYj2fFTX#{+t2jnp?d$% ztGnId41s<%9aneNr{B$|^=9+#@^IQ;I>^`l_{aCZ9_H!vr|N&Het!S!=KZg$-M#RZ z^XVu&;@A6WD|qeZllTAp$J6>@u0H)jsK49oeEEf4%)Z^7_Q%hB6u$lB``546tF;3t z57kc(^XfRQHuK?Xxy~nt>iBk-0k;q0=w08gKOWfgnB^-z4Vz{}C`5<5&JXK(m0_Nb zB*B8b`aB$*t#-Y=IUSDscURTtx3hS2^CMjTAi(4-RQTgF?KUE6&iLzRG^%HS`1xxA zdDq*MGav8lVKZy8p1qoMi?v?#^RIt!T<6+3a96v{W@qE(znQMD_wySk4c7Wu3zeMkHocdQ%Oy{wznrUuiNH;)fsZ?nzBvrjMi?rG$z->n}uyL%f3TOcpr z?bUiK-fBk#f4$q!>)TAz`_*Z0e`=HV`RmuydcWY`?ZbjaIL?K7u7$ci|V_d?VjKL_;*!k@&@+ZEP(ocGi&Va)#t~{b{Ewb zU%#q;{`))_zdvm@YhU?!8V<+B#GJXd*Lf&ygPa!d{=)HmRsA^Ktap#E{aoZR=WlnL_09AyudCNGN^k5ozZEqsqFr+lFdZWWYt*(ePj)DBrAtOc2n5o|Y?_3f zm#s^qBHLmkkr_ya3uy+PQ*WM5DyRr^+|il^f~1;xZ4EVP+2hs)dMH0E*6j!L=3by} zZR<#MnY}1l2B3s9zesFGSZ`Mg6Q&Yl6mplj79~iDEIi{_C@|$p@hHu2D%48WvT%Rf zEA=3EfmP22mHe5$i}4I1#Vjzz?oNrwgJd|R9Y;0a(WM_zx2|Qf5Y++&354MqgtbCG zzCpzXRQT$I+91HK#pbByH+0IZ!IVhNo=A9-EfsvK#_0|ytX1=4!nX9m;tTNagFS32 zqV$ql8la}%NFWbp zT@dS1T(fo%VMF4`}(_)gJgd7%W~ zV+u{&=)~>iYGq2w!~2W>o+xjFYwsfRNYIlOx*IwXD?q}rZBs-PdpEX;x^bNZ2%#r~ z&3izeme3_4YV1URhpfyJX^SK^FzrjW!?yW(AA^pV`*#IhNEWdK`EhxQrD7+D{`l` zoE!louy7%&s{sxd?Je}!g}p!(MLUSGNFCH_R@mO09w_O4 zv;R4$pNs$=A&oUs*<1M7mCVW6LlR^Ie3a!)P1A>p&D{6_THYG?Ih@0U&`brd}wk%Wn~ z8?2;vbx`hWuoFP?FYxN-n^Ni3o+yr3O4P)*R9_B4jF4J;P`(Q7U1*yECwM;KYs$YU zCsZ&I>nv?0G(>@&l$pSEv=%{i-~tM{uihU#Fk3wTTxps#d?f>hgr&m1b{*4C+n!EE zi0WUU*m5&E?ErSdisTJL{iLImoZL1tL49KmK`B3enL`3$w6J&AW(1?Ib`+U4)*@4) z)RPCmB=$=866>Lemd(^48CtNRr<{qsx4cg3KQTY5A-+~+!36~FZ#EsOQGp{M zAAypD0{z&{>A9o`*uuuOFmWh2dg=_JX`rb{jLq;!(adWL3%v<`s$!iENA6{5+>M3UM&8PyQ2NOxTy4|{!Z#z9iY~^yX9ut&gE*>@z1z%pKvEsw zw2|`ecH$*Fa!}H-dns?$7EY|66P;)%!B1CefY~+F^oJv&ED>#|*oIgD^6DjWEg~DY zR-K@m6mj(Sf|#Xm8fWpRluRw98ryzul{L-FU1k8Qu3IxbGX)qvJK*%jmWe0F0x0qH}GrZLOHkwOGl4PCw zpfE#>wNiT`vo{z}*J;FrUuZ*9f;sZ}7soQP@|7Y85e-SSb1oHGoVoO^XMUuP`u8Gm*-?)LWrhA#j8 z_!IHjjy5I!>Gk^7Nr%sPv*%#9i;U~MUvJjOY5z`8!RcnMj?*u??89bS&DH#7wr}3e z(_UPiarJ{=Q0%8u_2q86+U;-VtLmHM zf@2oKxq7{M_b`c%_{~DVn{QD)kq0ke&mX4aes}Wsy_p};htv5T#BY0ktwmUuJBy9P2r?uT!@YOr}uP@hjq5G+tPSt8NtuwFf zN!R*eZ?~T{ZJN~mNpJS`dHvmdGM$dQ`|X_P;C$ah^@nMHv)(SO$6rqS?c{$xpE1hM z0Mi*L{cI$kkjyvJQHx|+{c@%~ezm^2u~qWaF=U_NykD%ZH*1@u_kXt6 zenfFTTUphoUu?hr?2I^m1h;O^7kUn*zF%0e?-vO4w0-}-?u3nBACL2X?`U)Xo$bFL z*EW=!^?Z8z=-b`t%}y8k{z2&dr!Sm*c7K?KFI_%-hDiSK6~5l>xAPMT^DCP`5g!X4 zyx}Jvq=&X0Ir-w7_Q%8DEZg{l8)h=-b>6CGer#1gZnJCKjW@nw^rXU) zPnq)E!RhlV+pg^y6mGxLV4n8#-#ik6fA|u2YneE$#ARK+IdAv#Pz$7MulVix4KsA! zQQePd{qtp3Vy>PKwg>Iz>VR0U=DgOtr(wSQ{k)yutl$6H-v4pT7Y1~Pn4cMvuhpV) z%xwdD=i_;l@7Fh%79H2mA7iTa+U3`~-7gD4^!9XE?bi=Sn~e;nzp8!<{CBVu9qX8N zr_LoTH#^=D6TFi397rEx2&LvhcBGS-IU&w}0R^h0PDj0A!jq0g1!O7lh)%NXxpq<* z@`!~p>Pt7oUAguJ5#nSnDQT`WQAiXLH?&YpRF)9im`8!0k&qRnK`Oc+$vt4CGPI;B zDXq0Y@mtV*AUosO%Jqg?fQ3#PEto9xdMP|@&G#~A?&Et5iUchb{OS^LPDz4IK>ea3 zR^j!!!>D#S)GAc@oZnp~cT#*A#3k8NmfRwz)Y3u~cNKAGDRkZ9I=iM{|&s0~RZPS-XJB8-G+!vgluLKS%ImAIoEpg=|=VcYs7 z0&P?w9+d5eoohSVmIRguVgyBXJy;Sn0V1uT8!T*vcej|lAze7tw}yhsI46pdR|+ZI zX=)^`GE?L7S}=(ghI9kWqQrygbBU{rs8Q@dnI2LfB7d~t8+$2YQ>1MDREXn98N$|u zh|46M6y|D@C~Nz=B)3txY#~!yq9C)?@?aKzBNa^I6mWfXWM{XpnB2N`v@pE79ej;i zuPIs3B&yNc@=?{3aDDc1i#(-v8oLr3Oo>2o1mIJ{YG1S^`|&=+7)LamgNNXd!LoM` zMH5X<{M(uUHbiAAuz|g4hA|d&Vfj^hX_i9BEch>0nW8jop`s#QbION^Jt!c_jo8au z%5L;Q&JYWU*QE^RWj+*R!6W835sI)d+kPo?xkk#1o0BATH!XX~=E3o9ra6hv29r?@ z(sBstRp>G1r6LtJEQ|@yhK(LdG!iY2N->mt7i(3KNUs+r2`U3isJw*u z#1{dbSfWZTcxLqX!L|aDCVKH7ljxm!RrpXt8Vg1Yzqe@Uh=^885&61YC;?8(-pm|# zDCHg@s>6=48!7$hirhW!gg*6!P~ft0o1rMCx#yOo?^QN03EfEIWwzOD*1_MARB0$? z#KVg&bRU=H>fFXq`daRos@MaqPG%BY;yM|XmV@;vKyV>E>T4tk`%l|H4pOb8uAXs} zLr1=^8fb`4ca)Y06pe?3EJ(eXgjBE?`$HwKB$Y!RrFu%PedQ0{OP2%8A*PakU@ahpB~Wg^$4)*AEq4+GIw zYt$pPaS1Jt##>B0NK4nNWQ;dg@v;y>%u#&m%Sbw#g`up^W-F%R9+*lQy|!S*@-t1E zJqQ~aa4Bk;KrTccE*wV_+BFtgAvt2q46>6mZ24cGgne)=n<@S>P}U zS5B=@wKp;md|C+<3-ZawbG}wm0Jcz`3eBx7A2uurIv0wB$mMc|^w{-hOCn(gr&E_U z*_5`?u45AzMO_CtKnc&a4ES!$5Wz$?g6AWNka`Xi*VCmDfI}uaQSgn%I@JZyl^+*E z41o1e_bc%i$u<%qLS^HCc>XSFJ0hK)oq7}|=+2=4A7$^P5sm=7wU+f{YB@$I#nlR| z3iuqoL@f2{eFgPG14FlPJElf9HS*ek=Z&O(r&KuX0IJ7^-Bwm~V{-)-%A&ON*dx+Sw zSsY5~7~iCw{R9Wi2DwlI_odQkwGd3hg~%w~)`)?cA|W~V2)Phzb0LylXM|v1q|iO6 zk{CAqLXJ{F%8&yXi}8ydrR?5^n5fayY-Zv}q3as;X_UVh)LKlc)0##s@P{EHkkqW1 zG;_$3w`%DpAI1hgjD*k;uCXg{%}aL4;(U@vkLI%@p>-b|rU5VC;);-#GS9nc;)q$U zw=fsdnL!DBl2eoSB9)!ui{z#mNhAbA39>mV}}HRK?qGEpA4O(-ce@FO)%{_i$Tx$AY}IyK2<;*b~ZE>EE(@-7lqfIoxwngQSv++8&kW zC%dAK$R3V)B_RX0vZ=EMaev9#gdlLR$Wze<^i7SNZ5$g3u!7T|z7uL?R!};CsSCZD z^`M7i6nK(WN>CP=_%#C}k9(o614Jc3Ju2oXF`a)H3kg3zMMrZ;_%IO^58PwPO9e!) zH_=?=L;$iO%~c~2L)(rmFD@~4kY3f$sjx!Fwdk`7 z-X}Li_NW~ZA#_PVa(MG3K|)Hh>$NF~sVoGZij6|8j(imfvL*rjUDm#Eo70z`#qaYz+ajk7q3 zgLfY{&p!Ukk-z`1z@vQ#!+SOT&Zzuv5;$4i_J!r$py zcbj=H9OX9iZi=YRALQ`~pTL5LYCmtz2*rC{B47Kz2dS(+-L6-^Y^Qts)zAC)e?C;d z*Vi9G3qPygPP)_m!)897A0%~^V`lRh3u^ki-Tiv&_`@G`my-?c2ao!CciLCCo9SQ! zBgpSFhS>e==-z|i)6=G!R@2RV{{dn&)2XiR)^DD{M;AciZhyVoZXf6UJo1jFw4pz3 z?MM5=`uOho@GlqmW?M}+H)}^*F38Tu>p7dN(?K5}AMtwccwT>oS2jln`{r7R-7hyb z3=385rNgk-y2$auhFz%D`FQ!w_HbMuPqubG`Fww@=KPP<-TPm6o87J0dViYtHrKYA zUrp*>Ps124!12ud`fOBwxSRI( z(`r5)9rpQzt(|w`y53bcQ}y@u<8}3)pMLcCR~a_?)9P-w+h`fQa`x>HaJY`cp11FR zt$sHjR^o*{z@^R*@Q2<0=vd=3mu=>y-aR;y{GhKZdy$0=T-UZGwOg$--2e4%cf6i9 z8~>B@!#NH2@8`YPovW{}zPx&Tto`Zs+A;6;(|pmNj?>L~*Gvl$`H#~|`1RCWEGtWx z``6XmJDb`ke&#=%^zrFZxgZQqiFHa8+L_Sjt`^~mC7I%7*{GR7#h28!ADZ{_>oBwX!`dWAH>ysW; zcR7#L#Rr4n1G5((+KR@0fKdCNn>%xiP@5Q11L^=M``cKt& z3mfx!7as1`4-3rNFMKvLYoYy*)Am?gDsB@?fl8pxs!TbQ~lv9*qaUFOeqi~?aGA7^< z7WOR$P7MYf(gPU~|EBi_vc072!^cI59MamGV7<8USQiZJS27P(S{82X?H z8MWfXD+S{?hz=TZv-VVp7;c(cuy*nbm021J#@<{NX=3tMJ~c*LAc_DUi}E8Dh3nWS!Zld@+k@X7q}oeyLhIVZz``d&ZF-4LlCH>@)|fX; zJjsLP8VmC5oRP6jQjWP=CT*^Ws!;(Q34^tg<|TS0)%eOhiEiGDU<@lXFTp_Nz!QsN zCoB~)-gQZ-uyXdq&IxGB*7kqVbFIH0Z6P3FqwR%GLTQ#-$tDW=+i^?-#@8uIIg7Fp z7C3!OHC<<(YkPIVEsa5jfzY>6&ES+@wM}cQmjWl7*s+j<%X1iJId-eWgqXutI#KkE3Rg9WgK4c4WU77#k>!H*>xTgw$6ZpoP}j}1 z%9u(zUY99Csv+Bmk(O0oi=h+RB_wpkJ;JVgk4ZLHc%|KA$DrC#umlrFWFNP2ste9K zT}oMvqW9m;_#ogs;%mhXj+{-SZ609Okz9Jo@|2HNHN(K1OJ=M_W!PTBNaAPaMoKD^ z=A>lO&dEP@l7ZvUzJR|N8ZopXFu<`bbV_<#eEF=6c;*P0Dw0m@FY84PjpTdIf=m`D zIHd_;aXM8NNm@g-35MBA+O47E+o|)3;ii!!+Q3i-LHrS2IogIMkXKp~Jgw7Y~qk`;(R2=Gp5z)_lbznicSNoaR z_M>?lNZK*>jpQYe8`XY+&`LFn5ec&@y|z{m04A~v@>Lse;(LL43vu0zSc)2`LJ#IA zG&RVVwvu2TU_vaYuoDR>*V|YZy=1;h)oP0J;)4tzvBF3)frehLQ(LH~x;YzEr=)d4 z>eC91`a&)t-@zuN2pV(Usk11Ej%XFC8Y&rUM{z&I@IpLiBNB*D$~i*|Sc`i>BE}-e zrPXnP@KA+X4Mt)1Od3eS!nF2iFp|NXumNj+LPYfF=Z>vN<;ZpEI;ATpijX=00k$QE z1mUk4>=x@tK{^=(^6NxEf?SYhmr*{c8ym6f5*_-W)+n$+W@rM|ce>V%6V&@uZVcr9 znGNVVR38^Aj=G6#aK{0KSce!;lhGSTNo1fAmaM}qDgYLOe>*XEB3@cl-vf+rGm4!W znedW&7-?htK&m>Z9@x{i z?j=@%Dyf~2?L~`Q_LJ1*Oc@g?zmpSeOT~^dD7cz282W-4JuhN4Q-fQnZai2Jl3oC# zBHG)b=o5Fd)xN+8--=le93E)`jmXHB)PhS!#YV~wEYgn>Z!;^VOA`X29pV8fbYpWq z%BhyX1}keADAaGoa?P3z!%#FNc|_I^T|t_dlhv9mY8+KkS`jZrMU6zVJ3QrOS^z%U zig*}2+DuQ4p_eU{X5dESZeWfJVGk`n004q8(-?zD1|VfoS zU6g!s*l$a_uu{IUqU|{(;}>lN#R0WcR460Zwe}75od-KI%iSK;6GTz3^1Vf)xLz*^ z#7C{=&eO_MC`ublv6%&;x+UTIys zLr)E~E~1L8Ymt0q>>ge`b)?=9g3PJoz z*b;8{Kh+UqCb8sOpeXhWIuoS76e(Y)c&6n0ckfC$Nph~#c3d?3U3&MljTnYTqWZy6s1*lZ%r+(EaUf4+S-`JNmJuzNglE$|*%+r%NE4ttpAT zR79ZK$W@^uL$no?RtYK-524h;LW zF51(5;%$*qSM}&^L6CTKF&_$)ZJtEC1uI@1^K@qD9Nez1R3*{(Y zK!gt>%0g_tTwro^rF+#v+1=X%6KVC;>Ux54x5b{&azrP;!4D$c`Sn8h`>@}=o)3roT8{U%hnP?M4?9QTky}o%4)NC#) zA$Bg7--qV>!|L}osfX(NbgVW4vD)x_G3{6L#@_Lw`eI)H;gJFM+uinNw|z!Y`*gQw z&-!}36@l+EE4D2^#)wv*F67F22=a3*oUmW+Hai>g!)JEKL-lsH5uFY%flVI~vi4?U z^`p=C)0bcCIjT>uZRV;EZup32{Y+!FpXcg7e`S|_TzJ*D^K>i9-TWeF#Oi6k7i;s% z{?}=*1tMBqAL)7Cid*|##+}2B{`&lswa>zEp2X&=`e3U4IlemWcbcK$*+Bc4R)^zs z`^cE=h~wAO{qAIww_d$lZRTp9F~G}1ywy-#R6h&T{+v;J^LBc7&?VQ0yAP1+KdogL z%xCWNM%atpTN`2d+nbMA!oOM=yzN1Mp0`_}d$qQgt?~gF`h1gR%J#d{?cL6nTz;s( zd2Tkof0(xy)tx}SA5xY3BiTc&7JhGsZ0|O@;_s*Zes@@Gh>g_pIA7Xa?Ptd%KjWHq zkiRz(R6lLZ;idwtGn*tHJ zFq@cD5y3@qF&%#*x*^&+NfCkPVaIsr86a1{{}8V4RUi zmsR*g2w!v(W591VLZT9IRg@}3ERxKy6lo|QMGHS$RGth{gl-5{-m7rIvhQYw8Z1{z zBAX=S+oX&vacin67;0}_9HMXtkaaKzy%4zZbtj>A6bNZ-6Jd2k3SC%oZUn|0sg)su zB#}RsXi}&G)dejt!pBRe6R?T`IF~HDnyOBGS{N8AKEOtB5fVdG+}h5??(HgHv zXx;!CEXH)^*pdD%pj!kYOqBf7Yd6?Z-m0RC2^Z-}Ma(I=T3Abs3&ANDY3NloK~{ub z%;=C09^^lvEZU%&dX3YiK53LRweiB<{GlnVY`vhq2iT**ZeXq;N=ns2Qjl{o2AwH@W#%RCmb2{b@1Nl<53`SKzk1>!8xQ4~+*4`m}iP+7A}U&FLQ$l# z?IUINM{|L(Mb*N-SX2c4WN=O0@=EGBqM+A&W7|6jdNkLc&`kDWq=pU)2o!fx5!#8) z$vYj)V~2`k>+%nToE5&WMOHF9l7_laV_B@0hb1uZ!H-L1>WkQ;&{!N?N-w*m2_?1b zVw%jK26uW8$rzFM2O;0Ngl9Ic!==@m9U#gh^?atDSX5D27ywUOaG)%FKr-*y8+RX6_Q1PGtgW zZdbA}66vz-4r(KXY)7viT>v*W-YGyec=P_{Si*e8G4w_B<7?ZtC=z$s!O<7J{dMlC z7J!kKi!j?X#8Nov0T+>KCF^J}Mnz1-^w6c4C?zM540li14V1%#$1-22E{EPtWK=T| z%4^qZl!JJ&RkTV^oK(>RBVkT0HQ9^fxh^m1o91f{E%I-pP6>K9-)YsnSE(i&S^ z^oU z{HPBcQOkXj1PrzE3>j^^|{!|o?st`I@++v zAX@VpT^6#efus$~>{4D%0LmhKo`Mu?oGKT4I^LD>>E*5bEBo|YX*(ih`?vbM-r`+$sNLJYInB?$pjrc7i3 zqyZ}KeI>HW=Sl)-A``feFQu_YfsaWwM4D?wk-xVU!7N-3VbgCR>Z$RyOramY`BS zbw#%31@_1p+?(54)R40K6wQaA6kDT{tN;YK@#wJ)p;xCf6-0Vv-rA;r_W5sM`u(2< z11CtL1tgC=pEJ$ptl=Qnu%)E|Rb&4!QFH{bm2*9)NZo7G(X?QZqU{O%)w@^N=N`&_?WAMYH# zdb_pJc2x29UB1yX^ycFbfBnrD8S44PbogaHR_5ylAD0h4OR0M6w7z=0Cy1%P=b7hK zgvn=s>;K*GJ%f~9`#qk3-`~z}?BCA8JRkSFhuvm!CUbwQY~SwoBeRV0gJ> zr&Um`riaPFz!~`Z!+PHD7d-Fjdfp$`+pFsHWm6n?50@KTZ<~)7*mQqkt5?Lo_FDm# z^R4ZD&z~-j|IJq5+4iEEKD@-|wxHIlY5O6h{bCszUm7R9^z6WOA6Q?7kFZ^Tg+J`p zTcK$6fhOZ&wks}>U>kw2muo*h%j>tnT@kG6Y%Ils|q3Dzt2|cs~;CG?C-w(`svb# z)Aeb8Z6mCKv7gFDJ-^x`#yc;b7q9yL!*RXQn{5yF66@2w{^DW3exvuzciWjwuubW^ zFkIMq{q#=DWd0UxZ|ek@R3#q0+6CT5ttvc(J(3*}<4}1VSvV3JOmIlLjzX)^r9!|% zh{EJhc~zBg==QPKP(L_;27>x?s2xwN8{}pwhfNNJMWB(GER?e=hzV+^gQ|<1yFXDs zHi-LkaWbTqk`zZn$ptZ?nnO%LnO-?ziq}JMX+H3kC7=YU4U53cy(L!6)T9C=O_=#7TmWz zfP=&&Ab1@m3W9`WpcB9=2G}9qiOL(PGuql5DM~sBtN<7ai%gWKpeR}~KkQC5O$?N7 z7cp&Yq0vvt(7wps`c6B6m|8Q2f|=x8>5e2EC4nSD z8mTl)=_IvM^hnjvs!T@o3ffIJEQAKJsbw=G3WXviT4%z)rrNm_umoVy9(Y!%UbSFbCiqmefRlY3j@>0U<+M?dMBzOh8A;~8NNDb6rY_MWU zI-SdY(z*wCXD~a5DPci7Y`uV6BeA-ol9sD)L9aqE5|PUjfUwg+hVoIVCboF#%G%rm zH)f|^QRD+0M}f>Rtt9qVDs3v)2X)&{s$}FFn%z|Sg-nr}*APZ0-d@6>s*PksFur;w zTEIA{bfJ-?3Re)78}*@^`6x&Oa{68;O=6V`^XdZD4+f=8u>&QE^NE~CkuhQ0lLW?z z8cu;k*{|bctnv(m5m861@@2hfr+VA)Ix;4Gv_;Cv0Fe>O3c|J3B{A<0)c8= zsGXL??ZBM6t#}P-xaT7p{Ve`vSd@Fhnz|iyEF-%bIk=EZR|^P_Ca* z&4Ada-6~^AWO}23U36NQV~_9Gyd+s8norTN!9A1;qIUbi4+|*2>O~3x`+QWH$sz}+ zYut&ZLw8V@5qjgXL*mS+oHoLDgL;oq;B096u6X%8i&5K$aGzs`bPl3b^9Zf{iPY#3 zwj%22k#>-s(Progl4UCk`z?hNAlBoq9rVZ)Q`xd^2t6D}Fq3o`BOh^WfR z#eK*H&O1VVlZvxylmajt)MGwINpI>&5IK+b$rEE?Hr!h_H=fE^?I89!M46jURTngv zK8Y4bkHt9V11|K12byOwaS$SD~OA=I!9ANoRHL|?RCxC4{ZIsXsH zu{286b3;n2iX}L3<_;CQoq5Z31IsEl78I83tnPNHdW2mZ!ogE%QYUj$@XXepyBrgc z2?e=o1G$N6m@e_JbtZhNiw*}joa)KrzRupF5Fk$ry=`qgwsgwcYl7_@%y}LFJ#L>% zKcJiv(F`G*P_1|Z74Xza9iVqgY}z`}%JCHqH32=lq~Zx>zFVoKp!Ad!Vh}vu^dNPx zEFm*d*zP~bo*JX z^rvSxc!UyGpTC{<_Wj$n7(CZo=W;!>h_=peD>~7C|EGV+z{tM6YX8cgd%u2oqWH|1 z$9q9EcNrylQML77=|4V(0bW!gTs8jH!Y-O|%bj0%Tdk)3;i~##{fi)|TYYI)zMch~ zyxG~c=H=Dr&AhUQSm_6q_(iuzd)4QEzS_^zalScIq1ykup0Z9p& z+}#TaX_t7t-`!V2^BWh&xtr+>8n_$-4$tNZnKxA%4P>G`?~1X};K+HJQsvRX7d zfyw9j{2qtDc6jOK;bFFK{S6)CSbh5Q@BjY7A*~1JBmPF9T8F%T^{0pZd^lVPe!g0* zC;N-T^w~va4{$H!vR}ytajg+J|Lpy*ho@@@mTC{UJ*{mRz7r<&53~L2=dXVJ?BWr^ z`61TRp2sn(oAtc?EQ2_oJ~+d?SG)WBlK{@oXR12P`!`w*dBtRm>%|#TU!^k))gq9m znbSjr3-gTr?O%VnuRe`+cNPBaU+>qSX;JtCEvqN9;OgeK3s1MFecs&*`1$pEZwu0q zskW;!FY|fbWK8ViQ_6hY?gef?iX?a?2=y6dY=7-5`|bK>Hj8jN*Z^;KE8(CY_k6Y| z%jUKK%kHmTv|robPS@%jurQ;wb+Xy$0UxxUwzopQ-`MM9?76Vx_Wy_BaZH-}*=p|U z&i-rnaL9|!0=BCmk5A|u{quwIG28So??u3UICBA?!Oj2WKjc}k7nt*6IuFBt`41Oo zr0zxa_RfB4WB6u1KTp$-`3d|1%yrGTw;=e#o1Yi_GUqm;_d9J0+xp@HxBc|gzK|E0 zz4!DR>3UWCiRhmPi*f5hkF9nd-KOml)Or<|CgQ}Z-QA`dlN_=TNGZ*NFsc(u6|tvk zuc|d=*wSC6C^aPtOIvZ_B!VS{QUSw#S~cAxZ9{BdP=b?0SmeUH*b<4FPuWk5)$3#q zS<#J}{SdD-@z~wG%ihVT$dh78CUNdS+-<&7AQ)Bb zgz_d|P}xdA-*ingHb~@Z5yPO0C*+P)JEh!U7kl9&v2QsQ;zT(SqkNo0R$EDRp zWPDa%ZONUJOBYE@BISzjQY>Nv|3R!#Nmm?1B4(vTs`L*bE^1Yu4J4DSH3}u`q|`lF zfI&hmESzKUeoYf4kU3L{D>pZev=fBtKuB}+0s)j7dc`t;8JD)nVcAv>fJiU*3236ny$XS<^Co)?vk z6NQ^ylOp#=v<;zOI!D-1c~VSjMN@L#1Z~hdq_T+FYUa1=+c6QATuFej=%&rTFk4A2 z3SvQwi8JbE$qKe%Ungm72ZA=VE)D=DS-io8(6py`B&v+vNuGcg)vC{8|8QGpopk|WAQ&>QI$xM)HF<^v&>OSY!olw3vBb| zI{@~-6KET8bt0p}ikMduKF)~wW2K=ck?&r7Hihqy>$7cJU|RCX30rF4kWf=0@$FW{Ca^$O_+zQMR8Y8Hb}H1B&Zj?aFD*^X!w%vU#!dwZ zRKjRU7J+c0Djy+JN79>+*=!{cjMo~?`(x$et_=-m|A$5_CbYCiiPS>Qs})%Xl-8Qq zHi#x_gUa#%EgRKtQuL2$atHH|;IuLeIGR^OYq773#9qPni@-Fr%NAdM+f%5(*H8y%%aZ0{Te$rj(nE;?JWRs0=o<&NbT8;JTRG z5>r6|lUJ6VVjZy@MDzxn{F3S zO%!q33Xo2$8?l!XExcjlDR#y*t1AfkW5rZWr@qQ_FVLC3RFd%WZ4U|kta_oo?F{xvQs95UG+-7iHN6DabH=MR3dAoMFQ&wd2+W{J>y) z5gw}+{3*f->w*jowd6#WJDT0ZRS(O@7 zME#^rZH2V4qZs0`ORtmmcMqum^^KY%g^E`qJBB1$vMrhlKL3n)C>p=qsA*wSZ>+FE zO{7;5s-+`fN#p%6)aBHOHJbc4M?H`i-bcL#l{s~qi&hyfrj<|3$#$_~6vI_Bs2gV_ zuR6#Qn!Tq;qY=#!wujm}rVcvkA&HG1c3skzB&O4nX28!AMF#}f3by3Ump};`MGha& zZ@9AQ74a+Ovu!QbtjQpcg~W9&ByBMc5PW=8n<5NXPvDL@v*My+1RyWXpB3{fH$0}&d-265!GObiPU zwy3r-;FJVWLej}Jq$Cb#$o&oCd4|ZoAF$g6$cxv<_pP^o_)Qcsvd_{$SH8n&k4^FakP1g`xq3~-%A!_9-YiLw@0lTGa7OnSD zootNMwwa=InMJCF8b-EF!deo<*p|BFeIT1OI{h_9Ix%Pol|NC}59QvJ_-xfRLH>n% z=yWlO0%6-xQX~s=Vo{Uh@F@n0&OS@wuNv-2G_)J-EKfvlu7jUq)<#i$(JbWeZ)`XW=2liN{Ey|7KqeZ+ccW)iM_SJMLzyb_Yht4(W~nqji$X47C@E za{{SO>;#Q8Ln@yU3$dgidNdiGdxr=U?nlrZZLg9Vq1p~z(}1K^qrEn~o*C*vVqWE< zwWXn?GVHUYj39Pn+Zh>{u1gIelvfq)LL|Cw6{*x^49ncXm$VE;@Pp0%_F#e1%66SV zxhJVp#GtX&^DnVv)Z2tXQk*(KFw(`GYQpXcu`s_(^ly!@9x)EDz~dNZvL|Nc+^^0Q6w`(MxBo{^u)$Nw-d zw?E%Su<_mgCL?pjxvT|b0W&O+O7edV1S3$DK%aMeppiteYE+r2l3;JzmA%Y_+ldVqZE!d12RGEp*gx@7Ak3JX8ZScUGi$46X{){X+Lz?vku3Lcw|i^v^xqj|L1~ZYT|QUy&Fv$Ke`|kWgRx!E z$iD3sKIL6r*!H;3ruaO#j`kfpe~9aOH7#J?^HM4X!2doyV)I*v@4j7Vt~*$S&sU!o z+lOoG{21O&a6IpHVcO3e*!fRC(Qr+-3s87}p)pw)#;+ci+U17}?PQnN9(HAO;XKsq{bB;P0yMw9TnXj8Jj{6yzuMUvIs^273qUt?odYr8 zP&Ain98kkez|vTra5<(FA`H$Yi+O~rzzR}taUdo96uJ?~&L99zp^oT|RTD;3gl@%{ zgU*5&9Z4t1oO?=4NA=qi+-?-Ui)ullsDfs^B#Z<`-(of*K!>0-#gV_%u}6#cC}AP# zHsZhOg}S8R)Xbf^frLm(5>tog6+;uz<+g*Uyr@%5F$Z%NjA9=IuQ3d2%7XA-00qRG zHOZ!sr85Re7*lDkNx~4&UTAvhRA9My^lAYpUbJez1uw8^V=U{~KEf{KNVw$`Eiy+R zn9=o0)Y!XFAdv)thMJNzONdcs^Z_Gz*w~A|1*#OWjWwYWBveLh*1~NJ$Sv8A%6ZX# zR!Q{Zrwvz8G^k~dc+L>UE)YeO@H#*s*;{NANf0IpQ9(4bW=v{ibe7HrYmnM@O9;tU zgh&~hAkB~t{QKTPXh2zD=2e!IG7O?rCBe&90xx6&w60n~>dMXDH%S}nx#Vu~0r49s|9q=J)$#WbJJ1BppNg1L_p ziNxbGrGf`KXPcr*dO8Wh~4`9CG7Ye#nnqHzhlFE;N}x zQ@9U{E$9fBsA?8s;$s{WBK<@(*c0iosIy9qxNJ^^$RW-VH6vgSD@hp~3!-+s1*)Fd zn^PeqN*J|h2UzklU}mK;UP!O{OXzGXa3ntJz3|WAZ>SJj5Nf+|62^qM_Ei6G$=W1Q z7Sb2aph=qO>f9=D(b;gdu`8Kv+pE+Bb2eH=^$tSRQR;GNeT=~s5{{FA38ScrknU-e z@C22NLOYJB!R)pqhpDftR=&MU)XblBW(w9tyEWA?QOmoU#!?+h816M!OMJI9b@}Wh5lv>ZCDEA^^5K$eUEEJd)uzw^u$O zGGGlgmo?1*j7VE3MVe6Z(X?05Nq-_5A;DCzk_)Iw(iZU zmax~*3MB=TZ*$)#T9D=fj|3!%(!C^mE{6J+d*3^!(SftUJj52C${|M(qN_tWA-jxy z)uh6L_{eKx!%hjFStMG0v#RDCgSEE^4}rM7d2j`=G}qvytx2t+Ht03%Qn|o&$+w-8 zDUx}B<5Kz}&^;n>MW9ZUP%BwQ>;*wEH$d&C7y88dD8GtG?9J68`zK8)Y|&8aRhM*AgHd0WYeS z_GC6Re*+Rc2T2z|z?s_8{??g8+BPAaRfELc5V0j@UaaZcqzLCl6l#LHqUc2{k#;08 zlWaA1ZpTqe5|22GkWqbH;BThT7gaS8zS_hWUHR_H%gVFcoElE_2x2@SKe^=gP%^C+ zTQeoUx<0y60J%!$C-y{HsT72G-*BF{RpO zpcXngPaWiEC67>=0^cU)P@`hekc3?&e;`KnI-*p2uR6vHT$U}df!ir<2L`qMN?)CB zZ%+7tIZyEmv<0(V4vWSpq7m{GLL7=XAY;iC=&%P9x$D`sVnnvk&LQy-w?y?Qf>D)Y z6(FoOSJ3tfX*64uJhFFO`~2cJLUKc6K~y^cdhITNK|R5p@LM$RqlyodQ;@0aA%QBS z`hd=m-6hBsD<9bWmzGlP?kbe6fc}k{M{`5LNT?T^S)NM_YY;Jr{D$%bW>KlcTT8J$Qax&L zHAGm&t=h||0Z4R?<``fq!~*Ps<+ApSsDe!JgS~YW2Fn%NATp~}_C_Xli)W;oS30{c z#DOs9G=#bUlV|Ip;DAh##!G2Y3fj~q2+gKyDkS!#)&{=mu^?Z|*XLS862o{oEVg~j z_d^O-J7>cHIemqmW6^hz^O6!RB}37V*e%wHtpfX5L-RrkP6@E0yDb|d)h`L>9Jtlf z7K#|4%Zc}5HqqZ$)K^dlouLw7I3LfRMq6}53;BzcH~WZ zMEb2*F_D;+c|A=-H!)Yt;&J4Qa8_CH20Nx@h!U*BM4JVpbH!3OoNzaJ75a>tL#9Dh=dXv&dov{ebP;X+?)DKi7i-t<49QBq#@Nv za}X?QFB*BW`!7{y0+QRuBn#CyDkRVeYKLKmR0gBeDX{k333i04C=QNhnVzvg{VeG! zTMW>mCW;PJP%yJ#rzgX5&|ep&MMX}{l{wl~jFo4ft>`Z#T>*Y+R6p8d7o;fMFX?$-}@ z`rm&%J;D-S?>3v=+x!gE!^3{}r}e!Mi4Rjo!-aIbyA%rUw0Fqjas7I|I`fSBozIX$ zUE%XMJU!}y@>_4WQnSwx#3yj7znT-EZs+1RJ10V2esjjBiez-!RyKa~_8?^PemC7b zVr}0(;)k1yx~>;$&x_`a@3VIiSFFavpW%GVGg9v#?5!V>dITJPoTX_cUf=4ie((%> zD;Tq$|BOIF2z+D=bkZ86OH z06H!6?3Y_$tJ~G)bdv#~U+Omtf$yuymcsVpgU^09Z`aReIDh4o!QY?ut3NIS{pGvW z#_qQa^TOI{j(eUli*+C8%ek0+CFUvHYgs%|Jo``_=Ep~TSRZWR>PKRfzNoIB7@H6C z!?gFYbSCAUV6l%lQyYzkyLSg6eXHvQzM0?R+Wyz`D^6#EXuqtz+IDw5W62*h1oj_! z#nvrE^<^fCw&y#3xI6QhAMWfETTu(vTnK7=xxat^>t=oZ zNq%_m2{_trsG3^a^-RkBNWh(muCGtN{*P6-i25TNCcp59K>4p0;_dbE-ADhaEoYv&Pz|i!k#>-r@&02gTrs_)i0{oc{=Y;o8$Vy=J~L*m8&uPP2{?crvZU$`gVfY((qRJ46*S&%2Vy5Zceag=wV&X&Vhmt;D+luCbgsar`p3|TKb^NuO`>rezt zzbfUAR!u$x_OpxW?Zs+dkJ@Zv`X>a6$TdiBSo-V;SXWn0AkwW?Yy^~8*HQsqqG2c$ zxHPxo*++n`IM&L66^u^UV5ow}u`X~K*=1`~`J>{ZXpFj#bN$U#BSdXoVkP~2F%PUp zn%6Ganui+GDDNA^ep*VshlC6W#zQj=QrW(OnPUdZ>p2Y6*89+jf* z=x4I?$-ziQ`I*E}W`GpQkGa<+nF?DqsIOzBmv0oybaeW5%61f_G&xR|t52dJ zW{Y|%y7p2frREutRRttC6rf;0AkU!gX7WUPac7bgp(|a$8;T_vJ6yZb(+codg9Psj zV4I&nqX^I`U#6DMaD-uqBi&hAUO_-;Y)$W*zEJsazg8F~I3e03 z860@7Mw;0jcu}oVX(Z~XMq1l-a1tb)wd6h~PcA}9wwG8d9N7w(u%}UI0uD>mPFlW% zPCp7IbTaisv^fYXQSgsuOD9Y(VBkb+fDDzbs@g=rE#Q_tSn<3fQi~m;(4vT$NC-;F zL(=+H33n_SmhvtfQcGC*c8HM#Tsy_7$lgDU8Y{5nYEqymv2*Dd3YMgN5eUj)9*m+W z0&%Jtg2KB_Q5=iPB(-j1Ds+rJ`cT5y^{6Z>T`&?97721f)+V(mD6fF@12&X}gW=QF zitCzkY@$6Iuz}C{o=GE1TIopc+u2g$%0Mp^DfP=nCk=qfx1cnFu8V}1B0x4oeoEN> zR>@oJ=+qvG1!ssqt#X)<^>akExp2hbD?x4|AZa>RcA>6sJ2FO;F%gcmMWVRIf^}*@ zQX?+{Vh4?TBpQXmTqwfth9DlW>9Y}#Z-6DFKDII!>bDo|ZP7K9GppiPLqj+^a2}R~1!qlxjjvrsWgJ_CaRv!W zAlrbVE$VMJ4{^jwmFO-p**n4v>_6bT?rh{YxF_vSO(s{vL|A}KvmDGcJ^ zD9}qZG8jbp!Pps=KF0;{48&b>OuLwOi2I zUS`{18x69+(y~Wjnv8PE#hw#|LsxW4jJ`f1Lmz~PYswXLc$d)xBm!(&<>(9oXC;Jq7Iigd?P}6@>f;bC*Z`` zw^Zp1=HQXeSEC6Alyi^|Z>IU#$A39+_x}-iS*Aa2F3wobzWysA1m|b>$}FbB!!01$ z_y2V>U*61@8E{%TPp2dOR=eeIg`}I?8hWc+pSOUseD9Pydp!d>`B{-{(x@x(v<}C#pUA-8>zu=}^6$ zHygY9{VW*ZT;1A7pZfjo-rek{(`LOAS*|`f>^9T>{7g1`fAv}BN3FK=?Ph(uzP6E^ z?rq#0Z~Jy_-}xV3Ri9UXJY}}dGZ5Dv;*&GFaCP*D5uz|N6|Z*ZA)NAy=y@}?_50Jj zy*!-umkT=f`n2Ebhm%d!f9mVY`}Ln3vbqz`tUbWn-OirEUQrOqX9#2e@-S`J8N&FC z;CEZCkB4fWM4FqqXYKz^3t8;(3H0j^km+>S@BjQ}pE072M!u@P*2Al?b%7Avr`4Ko z>hqU@%>LS&`Ea;=*ljkByIgL#aN0US>g7B=M>RhPXa0Cs^D&n}ezV@#8oRXzwJU#h zx_$p=d*m+rg~h3_C|;Kjql6-EY9QI`jN5K-^Kn? zK-c^_y5r-Yok6V&FYcKv_F=F6@n9FsdrD7wW1l%(b(tFnZqN4u=$=P#x4Bs^cu_r^ z?jL+}+tY1)t#9lZH;;dN(EXks*Y@SSUsmh#xS68%Op?3W%#$zCXPmZ=nBAX62!7^* z*5_*t(?T#Uj_&8b^5?h3|M0MRY(TI}?8L_X1pa*n>AmQl0Qg-BkAP>N_uD)fhxzRj zO|F=G^RjcW{q~H9Kfj%Qxz4-$&Dv(e{;!^Ky*h2wt8hOZHSwSR@G8`wT|5#7uW#=} zI{XO9&#S$S|Msu{=l^m0pYGe=2I}q1{|~6|+?w-*LUp2b8(O6EGq0|VaAL(Flx!{e zFYKoBi-n?e${4r;}Zz%AR2>U>~LegoiSpm51+ph3+>&zCJkXciLpxBWw zZnI>D3x&PMLMO+wL?N)Uj(yQ|Sgy6_YA6;Ct|AQ;@or42KVo_(rJ<;qdEg`G33Zi3 zAWWfqQfn3T5!%!?1#{jKeW+TBEr!bEK$p4mZ=v2@+pGJWg5j=j1A-(Z_FL%G))}yf zU9dL1qMIOQHwBO|>%CZ8Dg6>_p`<9RRf{*GCG|{HbnRT`7@pQ*l0@~XPRTP&Zx7Po zBW^qgW|K6OjjatZ+@uu~C595>A+`VnT;slPo5DBF31g|rlLTSvLZRH6i!`k=L*+2q zf+O-J^VM4#6AO_4ukOxf$&Dnt(tAAxc3ox0Obh>sa1ZT@EOuA7)!#IyM{6(>WI}}m zQUJ1895b0|tIcfm5>wJvZ@@RvlgvHWfy^ZRV^8-iW=tc=R8{v_}< zWLK&{mqfN>FPu=jW_Th&E~6(VDRhd$Dj=JutpbWP)!A4pF?U(S95U)bi8nc-SrG7H zL{76HR2s5whny2-am?@zh*J>&t z681ti+~g05z4k?+D{yp;&REeuy@jr`4!C}Iy66sK3k7)u*CD4GJ88aj8RsBbSk9-0 zQBpmG8jhj_$39%uiOlx<-}H!dE}F|sB?hS7|mNAiPTcUzJvxp;L# zrC`CFWE&9Sq0NleF5R!Pjl?t&hG|YW5fc|la}FKqs|rgj83xK#%U(`YoQHm};5Z5-#WOx*!-QCdv3#!&sLE_XvZ!^y4-X@< z>eVD9DREnl!3Dhf6(aT;Sn06}R)(kz;mx_zCzw};uF0tGAk;Ayc^(cc3*tlOJE-%5 zq!?JB6sL?PPfJ*ioA z#CAJ{khC#_E0&{PMiJ2}HW`E{Q4Wgti*zP27`DJAlFw}iN2nbo=AhKYgtbnel!6_$ zQsEBWTl5c7oru9YNnsfeXVoDo7${!-Miyh6$3kHKlA||tqTgvw1Dq^EMUwTiMUtpu zY_>r<$&gO5>79@#Oij+Bm`uhV=rLWYm#L{byQBt?2SvhBQ+2sw52*{lQbGbkU} zi5j#s3Jd#!(32ogmKYKB$R$x}BNRdTnu)-~QfC%EtDRWKn}e0I`kw@?wC{yz*zi2eW?h&8&xkw_4Rq`Ug9Ympn!rLbS)u7J!KIO zV{cBbQdtLIap?QA+lny5PaeIwmLlJ#Tcp6@y%wj4W7f@a875eI)ih$8|F zgu3)>)l@#Zqa-w&6h9$D?a!L))*dph%rc z)1U|eKm`_5XPQE>y9V_B#8)z+_IQ*4IC>&}dUxY(xt16PLd09tf=-1R@g{83vM|zY zJguSFN)m90sSyGL<>=iIaIEJEJDbD_YBmjv7z(L?nm=aZ_>|lesioU0oOo+B1JDfPO2#p0lp45q}4S~t`6A!o|GSE-8u0?GS~fVwK(=>FT z(7}~eGLdi3K@lw>t8BN^oOqf7Re44sze%?yPO2CX6@${vkespLLehTVg-OHa?gXTKW~6Ti+(CJ?DemIw!5OoaNu;<=HF|Qi?T?j2YwgXanW77oI4Ti1 zD8ffleRX0-y0;YJL`c)W#<|q4?(1(c%mVsR!XS(084JJmL4;Wp3sOKEQPhA>LFFvL zQ$j9Cfrk1lLRP~W%&s~$k-J?{=!BxOVoWxtgN~(CGjK4OGZzvbmG*s(nv-qDOa%i; zDNtvT$`x|bASMY;3JYd#6!@I_M`P&A>g~Xg6gTxVr_2yt2@odq8jHHdVOdKKuU04| zIMGJ>j9#VyE0i^qMS+lAiGh$a07C=|sZ{V6`#-he7L_frTSh$SMB-)UM3EIGOfA|> zSZPh7#6tGv7@*lI1PPTQ-?m1JgcG7hM%FLEZ4-Fvz z;bb31_v{If7c1dbK?~!b?9!=;as|xA)t1 zgDl>C`m<+?E5R`Z9(?>wa-Pnc)zxviKdIhd?Y5!>{rG!k+x(k?8VYInZub;RxZ7Oo z0U86Z{mUBxcMsA&8uHx2UnJO9K+v|pXU&+1lqfH^*X^T*|8dDW1y`_;8( z?e+V`SKI6ThhG=pt`^^a_;(x9;`QqIZe>$02g<)2q-O9yTKnz&FRRJ9|Dc!v1FQ-Rj!r)yFBiofD7l(b1AO_N=SJiDh?d&(lz$f3Q2* z_ibhx?06;2W&7Hj_1AaZFa9Rn^&rey$)Sj}I;o0Kl?cHj> z`xtur@aQLObh~5Ye%rh_54bs zUIZb39s$MLUFV0kh2@ap)5U$2Z|(iPKEb$8&$ZYrZ3cu7H{0?>m}~j+G!LgG`$ERy z!@q6y;T+E&Y9?3vtM~rb^5x@9z1Y|?dw=nKd0fBUni;X@`9h6FU;dTcwJ}>LO$axk zGds1!7I8?12N4JoxUIy;n$(hrakd~3kam`f6}sbgXpDy)EY}aun3}m`uMQErbHK z16#o6t72+0`R$PlOx=$Zz9i*D;Ng>mZmDzLMm+Z5Lyv?;>5)KbZ^~oG?ctuJ;9SIo z8_G2Hv^c)J)sPPOEXgYl*pF@-A`lZx61P&q?mIGwe(Wv zPITT+aph8>DclSJLaB?;X~cdnFedUYrAZyC!~iy_>M!NyZ(h5AE zkk?jHeTj|*H9(ZEs@t*^pJ6Z2ww}A>g8%?R6C;3g5V$^LWpj)pd9*d4XsG$D;_oJi zlLVEiaAAfcVFt$*V^u80KvE1D+(3OQ|ZgXiZ4tBzC05L7 z6q_;lxfTy1ilnHHp7%6F!Jni_E+_?bV4z9#64o z1*}taTnnE7_UIi((ML8&jRw3Vb39PKT($qMA)>`^LG*9fohFM;R#V9F+99xprtmt_ zw8#b!cXkjf9TCO{3&JC@DI`IzJCaF1N~+kw5%sKQ5Nh2u0omeO^R`iiV}?xW(l z^4~(~!2>SKcl`eVa_cQPI1! zunb_FF231>zX zC;_lk$$){W>I521>K&fBZ9%}34v6ZphS-u}wv^?ko+6P8_P7^c*|v29pFS7!eX}B zHVT4Wo!XZY0V0oxfXD&>oMEC4!^z;MaW62>2Q|XDH+Ho;-B0=6Z zCRq-Y8l7XLYEYTgNJ2hG1CA21QCpDMtt~zlSen}M=6OO{yC4l0lvs>aeHP%Lj!8!Y zybs$3*vdbtDhf{+<#JHtfa}YWQ)v#Rosy8smVVhtpiZ$cWLiD8naq%*kB~H~ZsMDt zAEQk@iEW5E3}DHVr05D|VM12PhuW9ScTqPIQXPq^A0V|B{8*Sr$e1erEJ%RscpYiP zqFD<_&Jk3qK^J)HYiPU0V;iD^<9AHi>2)vh2l9{Z6&WXE8?Q(qPG}cO8#i zbsknE>&tR9v6gH;M-GUw4gxr#OvCJi9*a(j5Q+V9)_my7*syZhN+ z;$ii`ar;QoD&+F*+HtWBHGJo|=C4;4_J@PM>Ql1*Xh6Q+KVp>L ztZ&y&Y<*{M^39359kJF@6ZH7jM!rA&yFJC$2C-_m z@Qw2d|7d@B|JQ+O$5Pbm38%a{CtW-H;3+MOy(y(-htL=y@D&&$OcX9oLV*&t!?|Q3 z1dxL@1KvbbX_$WU-Xq}750ck}{Di7#bC*_dPt@+f3TI-d!vdR&f+5H%%84UEIEnIh z3E7r!m{dtZY;(>XiHW<2o@c=@gaN^ofuY%2a3V-M3O-Hu zR{NCsf&|F2Xd*)*esp(;{A1BsaKx!DX{1uD>I{d%L9qB-U9t2=5`_CLnn@6rsRk^O zb?ls0ZVG`{A{j)^-%)~HRHd`{2i^86a2Y%{KW+{iv>KQj1s0rY~v^t(*}FJ|2V~Cn6^c$^>hHXRppF4P-(s2kw{vi##BzZ*qMh7AmWe=np3)$N{AyIH$<)kS0M5yuf}W?X804_Z$Y+REf4q zGl3@OaYE(e(ITcm2sYbQ19IL)Z*nKt{3J{d7R=3-QN2c7(qjj(ZRHZ%2&yQknQ^l)@vN`9Tb6g*dYnJr@P*}8?3J8rF zl|950F_%1y)KNW3UVEL%fzSa) ze7vc4GPoCQt}IMPK?#XfBdAOO7s~tw^*VH%{r;P(^+>Lingrg$$RtT!p zU^yajqp(1%87;tXfSz3M6&9~7*r?e_j?qL;y;`nMrHV1gaVNf+flv^6tiAS2n|xa- zT}bOkkR7qmi1@+;XP89R%4iIWrx&19oP0RH<#&6(1pqvuzWhJZQLLiDi)xX?|i2EJZt= z;9yp}YA+k$mcZg^hECa)CSsNmReJ`bW`1Cqx<_dt#8f-w=w|FkZnG54xdkv}>}*hx zlpoB}Gfl3#m27E~BOx3DLJmagYYsBi-9`2fhEQ%-MozO7(j1YpshX{=2;&idf5KB} z7V#_@$>eAVn*B60FRDx3B1^FIQZdEgJk|+W3oI6#=nS@BSlB`>$wi%VSgMZ&w~@|s z@fIL_P5f7M1gpEIQ+W4N)9h_PENvBVa)jMds}e8-JXZ5OmFrbrsEoLq%r%*?h15PjF=lW|4U@&H2Kb}jeSh25RSGW(S#i>nBCU|vSVtWZ&4hh0Qj9UzqHtja1< zf+>xuc224PMDePmWgAv}fo&IJ6qFGtk=-$I{#zj{YFy%S#XkqT5dT{&#coo4rN@6nRc~!wOntH7eCyK3-!-@?Z@TzxVEn>H;3Kg zr|s_DR(zX(_RAN~cdyrPx9|VVb$|2wpM~K(qijEOx#!Eb+tvLqmy6#vI@!C`?ryVk zG-sO+f)cK`SDSnL@ehZq<$m#Z>&>;z_;2?ge!c$i>)yezyE{G5AJ(__TK0AUp%y>< z^R;7F*UL9Ps#iiAFMelpbZ~Ci6ZG=s_5GE@8XqZmPmIBKbq7fbr}+rQJu~~>?RR&( z{TUCbzuG%o`Nix>+H+kle!sQ%I?T_`Jg0fEkM?@Y4^UEj*u&yxwGjq$v$l_}Ru@hi zytrQ9ZFaZI#jc^BSNmt58^Hsq@#P|T9c8yTN<=)}4*%m-VepflWqAjbFIU#j`~y z?Z558-`&^-^wqpt3MAWZU_XAdIvjR;!CCczpvoU#GhWV%+x2FnyB?0VCcADbj0f!{;fUur|tS@8=nBI=NBfT zVF79%_HT%fJJ-P#Uzpznz&dM#k`e*Jd6U2Yb)tLydZ zXZx2c;pyKBdu@wHauasRqa(^+?{`1hb?@)&#>bt%$)mvT%lqSQyYu~Ye;%%5g5TJs z96#-YIL()B!&`gCcVlBcoEGeF z4i8(xUbo)E{s~TbwRmgaQoF&2Uu|mbpa1bs|FRgz#o=yswO($_uwVPj+c%rv7Rklq zA~^$@Z5P@aIYXN7?q3Us|Ky2nM(?j|39SA~=<#UoAQ35;yE}=^1<-6F*a@iKolBAS zGB~!=z@&vbOD&|J22*p1I@Uy;{pN}0PWo7>AhmO&v`4WUU1us$BZUTW2kC<32Af!+)4-zH{Fq5%sULAC?5TZ;Hz|>ATi7cYyt7%cJ zIq4PBaKR!96nL<3Q4ML(XlZfsX8?G%Cg!ezuVp8Tn0&C@CyWsRmXsOBnzdLN2;EuJ zFb530m7TQ+4XW(3TtA9&Rq35bx>Wgqk=mEV`oy}|g0W*dL@ITi##zhXsDlw+qpcx} z1qiGehEg%z=<0n#CrIa!zvF{orxf8I0d*$SX-BT)E_#r9IZRizr~)j3{g0CSFz-2a zQT#=?tda;R9G+Cs_;+M(niE@%T@S|K*v^UZtID1b(r9pXZPWu6Z4BXBB>$+gIDDcV zhx&3_K{FA`Op_ZBFlv%CrAwJT%LH(bVroDV&!omp2%3~qLM=66uL=ZHqND7Ef_zVj z+mcctg9~!gTnUBKDcrUNpHv-%w>Va5y3poPqbFq_!{ltjKx5!q=yU>zsfmcCMO7ra z1`1=9p~8XL-pQyZlLdN368Id&EGdyXhwTatuJ?Xt_Q;1N}`g!L=zZl{{9#}-;CAeZl~{uiohi~^DtEO4sz7$w7# z(sqOzD(V9v%wH8B6Rgvzsy!d9K%RrTvB6PMg^%(uK~x4*R2ehl-jglsUJ1|*M^fiJ zr1{KZ6S;kqi-JdJ6fm@tFLgGv(EEp{EWG%45ksGRlXSPA=OC_euyZQJ!+3?xCfJ4& zqI9s9*y-Cglc8QWQVJF?)!e~#2G`y;Q%Vo#q{AdgrV1&p!eG0g!{GU22mwLGq8ybj zLoV|{Oa++V3$uFKAYPc`q(ovr?L~NU0OPA4GIa8mzSBmb;$gK{@r47JGtQHF{!qbF z!$>A1#i(X8QeUjaY%knGtzM{Dp_8y*(Gsm1>=sZSRnO(rK}hnYUWFM%1{Q||ns&aZ zXK`E+MeA%2k3{WeMClEdtmZs$<#NtS!=h597)y}SBB6Y-Q2)qUTct{v$R?U=Vy1~W z+QiY>A@YaTW{2GfQKjZW&T10Z4wWS&{*45wSuK$rM5rAURX$4pBx3MFNygl>LivTJ zsZhaq615Ccz)rdI<1|y|i;rTLA%iS|5+c>H>5!T$ob%HFEarm~(AAj9n&~D;S{EdQ zQYb=FwVJ9iqY&=QKNM4ieK?LuU9(K~lZjiV+WFB3dM%Bi-X(%O*lO#kHQSyFG%7FB zAR~!IW@eQcP@FonRIB;ht+{VYy9bM}w%1|@QS}e9#M@DPyVntF=RxgNJz8o><`3Hl z#Va#)TyJ`mB-vqbN0Y_0u%175|&!Qz=+hP-Vr}yA#9Qa0!1V0?8*?y+iuSUhYRYuC_qN6 zKI$=Hr-YfgL^V@M8&Lvjlf-1!v@JGnhYu;9Y{2#OHpsZ=7)6Q#mDB9&_G)nR?b z7YXnnq(keP*wbuTj}%iTo!02>*|9WG?`ae}X`r6}?5+Z>5<8Qrt5Nwps%xl!7XUL; zR#!GKu|&1x&>1SBgN}x|#WtVe*|upy&Ny{OWK?e-y@`jC2m&e*&w{Nq>==;}dd5!N zEND@|PefaTgs{SQ0_e#cw662}l)0aoRb>)U2%uiQGY@fPMHZ($^_56O6T)2!1ADcw=HdxK+67AGQgjZ}Trl$@0K9seVM`27 z(@xGt77m*_Yi#FDAgp)i}d%Xs>ZqiQXHK%AQV8y-O-gF%iQrG`EoR=89j!5Cg%&~~vvLXFbkf5*?f zM)Lu?ANYdME`}3ZbvEVEXy4mm0QC#A+|CmNYfBYI2nZ*SU-9iGi>x6*q4u}fH3K*I zMmkp5Z{+mJD9@Y6fu2xJV5Liiw*4#RD~w9i6wry(4qii!64>gerHCnukn3ff)q2~1 z5Ncs%LCHyWxzOfq>QJYWY*cote{@IL6LZLv<+=hG+5Fk=oNd;A3Izov6`+riocXL| zF9k7r480fnB!y0|!Y9jodF196n;rJKx4u(AizrJ;@=D@r2$i5kG53*RGrtxxRqVrOe$&Ipu)sM+YEg8XAN)h{ceA|w12+3zZ1}SbuZlakFUOc3JMi_@O~xs-$wPj zzZbRepEkGWL64u~e06AW%+D4dfsSu>w~h^c`Vu~x{k34LYoUZ2R&sT9v)yfWZ{IJz z_;I&+@BeLY?Dr=)>*@(?wcW^`_rtGG9^v%&i&x95&GP+6A3VTTAOHB}v&G$RV?((T zA?;}(U%YDna#;Mh`_k!wmv0+v`ipN~zBu3h<*QfrAJ6X(qRc%kUOD(ye}3^}|JUDt z|LWyeUw62<>w>7lAdhe)Qf6~?d=k?VC2==2j)8NClsn6?Hmw6!8ZJf%B zuMUUx;pm`g-&6l!>qOi6n-Bkfw>~cKe`#QQUG9soe!TeRN1MmxdHb2o(3fl&?+?e- zt=9JO0d(!Wy}sJ#kMY(GHZSz=-F|sxtHEr-O4#bd{hfg0LVq9hdN&Ooy}Z6%Z`TeC zb_ljTYt9e%79RO{UZv;Ggii-%KY4k--`%bJ%cnUIy8U5(ACcY9Y~LJK%l*~OGp+WY zZMitx@T2|u-@b}5E?&!!-ZTT~->_xWHu1y2?LJp4(G)+<)OcARVC2i~)@<40&hDud z`}6uBn&P%ho{*V8p;mu-?bC$VYdo$t7hju^_y`_8jp5nZ3Vk$T9kTxPg?GFC=K7s% zwe4S9_coV%rzTgw#_?u%|F*RuT;1>Y8m@@a{T-hMVsmcu=LT2&)mM)6zJDOEmM`*G z;*&)LMU^6@)>Pg`vL1_-QMDmS>Y>k<2~d%{5_2mus3JpDqw2>W!J9T`Gj${wgKMnD zo-^#^>)Ck%l!WSiB0!Ghs8a^aoCI`3#Rn;pV8c9@AaDZ(%cEN5P$gVAU@AvV&Ow4Q zr$v42q^3} zP?n%*$w0wzmVhNCaHJ9=u|yj1H03VeHe5D=rB0n%0~T{?g?F5tyMhw>s+3_$U5pwq zRfE{iXsKAhQn3Oc5L4}L(&C&TiWABz717&}M%SdiPAK>(XhJFgDVkD=ZW6O0rop0c z=}dPjD~OO5!8<8I64IOs!9mSz3*t^`VTpG!B4f6PBsn?Bn^3QKQuUCkjbV4eJ08<%LP$!@7!-KiwY)<|TOC^MDQKsPN&ZLHF^DzVdR@O>+tbVyQ)y#&>8 zqpJ5EX|+H2IK@b62KM$uBSl)gk^`sCt8Duqqi-R7WTxs8sE-nYn)L#kXSVNSHZW^n z0Y#rsx&h%oLKsELg^mKjKoZ&u+_g)V4bDv%Y$B;ePcT4wKw-H@Vl9w<-+-7MiH#+a z6&52@pf@5qKhzr$*IPtb^Ek<`Lp^76Y9nP&RS*Q9Mu6^#+zLC`1Pba`mV`$K=&MRm zjtCea@17z=VNyMYs+6{RA0P_$|7Ja6gL>~nj%F*_~ zr(r_8G>2izy)=B=%&Ciz{74hSq-})uhcnU50TE#0ZW69+C~BP z*`hbhGMI*N7f#jY8d@>dZmY@`=dCJ59J*%6AI=99+bbkFsTfovY z2UyvpzD^=KRNqfWjOBCXG<9&^3T8p!y^<{%(F{^y?Rs$iBpz+J1iVrNL=DSQ*~Nx` z^QMY=0zfJ8v}Mu z*jktob)wFFB|v)~E884qm*_OkL!B^OP1TH9r_|L_W3|{~#Js4IG}ulz2^vhOI`I%7 zU{M+!GwK7F`5yr{XwP0G&qOn6zexb#7vuZfOLes5y4uCD8^6RvDGULFMHdywl-h9v`x;7Wc%>nmk?%#?+~KOC{f1LqWE}|dtl}`I401rW z(>beQhI&3Nbn5!)+}WA14K1npgbt+1R|B>U%)<0yRz+Igq#z`lMSZ0qdp`kSA}4hp zlk;tkOH53NoD~cq6N_|;$lKClsGV?>n`ZMR@%HAFMl>7Q0yKb8svorixICJdH9*uT z|1WePuffw?L)WD3B&)lu1=ALOR7TD;K!Aas`(GL$S6NxzH!g? z?I)IM7yU0ZCj&!(T_*vY`B5hg*;=_fW0zU?M;XV`C54-llywL}Or-86%vvb!qGMl9 zU?*(gGp`;vihR_|LmSvEq)g5yAH+)~opb2(TWw8LB{V5voTX<*tuSeHRS_%^OF4%6 zHIo3?#KU6~AA6E&|3EMgh{UA_P{|yryVp(+aCT=2I?`#QUVCW18dRmlehi8A#zYHG zlNAQY+p00D({&iUM_ZvPDwL%-cWMZ34HTDAGk=)OO@nB{oc~y;NIQyFd4`BIi(tKD z0&ANzOC1~Qh)Kv0D2CXcHXBVfNnd)zgk5HDWb3b0J!GTGoJ|&GE(S00Q8=Dm{+vPp zVdBXdwe>(!(2{U3T~AN5_2Z1hGf|U85-&k&6|^J`t^yuW*@YA*%^@m?CUM)9wAO7{YZ~VmHoWAz#%**-|;rM`Pe0GLvZWb?g z0x&*XJYQekERX;3Pyg~_v((j}ulC#J{`$!UAAaK3`0(p$yZGH|rz>ClxVybuU+?by zG8)X&fjnO=@BiK&=E#_-YT2%kr=m7Wezz&w@Yh7iLuIK)t&-t>Y7H^=lD0@9r7~`iGYb9)Gv8jdQ~3+c>^k?tfa@{l2=lt+jtju>0_f=gaL+ z+FuV}Idks56WDq8Zrl0}1Yfp!y7Fu5^H)BBU)}B38_nMI1jxSn#kSjF{j(UGbwAst zYumS1-ZF5s>k2F{DE{A`V79(pA6CoF`tI<^#;sSme7j$@egaL@8`}okYkkeHe)V+n zpIr6TYUS@y|Ni~%*hc-w)tlAc0pj|v7q*5UR^Y4M{r>d#-`?*}Lt4uAI8XR5KK+}; zX1)E%zx3CjXce^19ru2#mPuLyRsIy9b>eXTNff9K@^%)9^a2v@F3uL%bc{uQrl}K{ zWDLiMk`vOgX4!Vh)q=YsghH1x99j#KxkPj{swz%1Qq32w*hgkATovL$1eyz-z?i8H8CS~$bxvqh{3 z6f&kd^fV65vl0joR+S6ClG|f*K6iShr(nuf4{LLd(|`m+j`sRYJVh<(gZGzNEJv03 zF!nWQkHDngOKr~L$creACGac-6j=gL*fDLpcIto?9(+!!c2Wp3_)6&?WL9brHi=Ei zR7nt4Dw`cR?mY3K>jg}giX(IVjFgCIwQzN%YThVGsi^V0V{;i!jE#`;$s&eDvcFn8 z6?mU@0qiM9GKs*4iriH(HzmO&2U9WnQHwrU*D_)LO+}~DiQ~0kX&x{pQrI(gJ<`4n zP;g!Sux9*_DA&yNOyP=JEhex^7FtP)2G@w~bEJ3?!Rpkd7~Ag5fyn5x60!*Gw`jZc zB!K6*dP2fIv%cBxPmD3lNG?(N5j#fHWNW26B;#d?&f&=;2H7aX%;B~ZIqJa(~vZ=7?0>c6snGSx$M^JoB=qI?UdM&ODC<5h@$N{U^kJH1=l&-v(J7Hb-Mw(wK_XXy zdG!i4!=*1fF9=VQMOQ=|yR`uq`&*36VK9lCb)s&Sb?Ej#|r6O=7xfh8W&&01Jec!2LC2O}2LDB*JfW>t} zp;ZaWD}!+~U#{2Q^EsAl6G>?xs7MRvg?$?sUYt3)*SK?6)@HEcc_B)oZhBk7b*zZe z80QKK4}&nO&S76iz4ghLG1P8V>445-Nyn*%&NBgw;MLbqP}dVfM|O5mK@#&Vdb^aI#}> zqy|Hk?7?PDDtdI(%FU;c^+oo|gvX2eYciYuB&xrzU(i>BT9FIYd{n>bbcftM3(|rD z7?lzkp+ji<9#JBJ1}1ILfGH=7XTuyRJ~dyOsxeuT1*q<0PhmI5=vWQ61OpeOd_sMK!-Z|NY1 zG20qJ(X2_0k8s*2_0l1%OuWL>W^D6qdl8K9WWGtphDhOp5@NMPt7+#Oh4MmqqenH0 zKaH%cYJ;gm<@+*e&I6{@14;NV#5^{Ql13A`nG&KU-!`u-i)v z3D~V2lcW2oDmlpx0scr1O6^j5z8=D8?}|)~8Xl(HvE&a9&aoAbDID{>vG=jaoKrX? zeKW*in0ID#iM=4Ud4#BhyVw*5N}0%$Cz&fH87gIE#6d^yVe=cYFR@Cz9U)fY4;Wyh zm;)&7dQ{|AGX$(VQ7vNUjQRI-*AvxVW2n+|fs~DAizc|ntkf`aRP=HV9;(!wk;Psz zHVNCp)%Y&`K3LyVa5=)>fw5EE4qq_Yb|eju>C}enyJUUSNl=-N81d$f9o4q)U^S1x z0cUZmZBx`?8bzr`bnR0%50MhXS^jsY(&zg`k_wc}wU7i=phhLckZW4XACyKP2ez~l ztF9<`q$UxnKdI-x1Dg9A6r_=0_9H|JZDHF79n8I@iyhOA9N%UhX}_BjF9#KVF+1@m zSyb}^p(jh8BjnCv(R%NxXe47v-*@*D=1;l)KEco&?`V%k^PwK{&H?q6GbhwwQOJP; zQ07O5fmOwYn?jCZ^YX&fIYDg(1NRe~6V%mg+D(eoVE`6svD=KrOS7;Uz05M&M#rqM zq&=wV-MoX($P%hO5F3z%(4t5sd%O<8f=;Oc4{(uH%w#U3>&a8+@SsW(d0Rp91B?M5 zMg4)kMNw9bNP%r+U3%ZwW0l+tEFuLyT?wH*WSB}vEAI!TCn5VsRB{{PY>hqsQ=?qH z!K&{7@|P7=45cjKRK>>5^%mi$F_~(sh@xOo#3)oEXFy&GI0M2D2ciQAcII~+pnjI0D zm~yXo>SvOukcqxPXAi4r>id~Ir!2T8Sm&fI1OPxWH`|jCAJ@QgSJ9KjkV+}@Jays{ zEupC1Vu;yo!=;MgfWc`M2I?-6*49Y60@5fz?pa${>O_Zux$_Ne2|<>AhQeT(EU=|b z9Ru`->jxGog->y!{`Yx8?0YCRGo-f!uYLBXk7`_qvX0`#p!)b&C;3REWu}S_l`17E zLcfp~8xfprzOqUbpMFv=gAmA!GFL__baH&Y7j*m)=0!FzB9r)}MCbt3n>gEak%qAy r{Rwrz#1=@TFKV8td7Pg7{Rw{dZ=U`||Mic5^N;@vP + { + "service": "test_service", + "service_settings": { + "model": "my_model", + "api_key": "abc64" + }, + "task_settings": { + } + } + + - do: + inference.put: + task_type: text_embedding + inference_id: dense-inference-id + body: > + { + "service": "text_embedding_test_service", + "service_settings": { + "model": "my_model", + "dimensions": 10, + "api_key": "abc64", + "similarity": "COSINE" + }, + "task_settings": { + } + } + + - do: + indices.create: + index: test-sparse-index + body: + mappings: + properties: + body: + type: semantic_text + inference_id: sparse-inference-id + + - do: + indices.create: + index: test-dense-index + body: + mappings: + properties: + body: + type: semantic_text + inference_id: dense-inference-id + +--- +"Highlighting using a sparse embedding model": + - do: + index: + index: test-sparse-index + id: doc_1 + body: + body: ["ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides.", "You Know, for Search!"] + refresh: true + + - match: { result: created } + + - do: + search: + index: test-sparse-index + body: + query: + semantic: + field: "body" + query: "What is Elasticsearch?" + highlight: + fields: + body: + type: "semantic" + number_of_fragments: 1 + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "doc_1" } + - length: { hits.hits.0.highlight.body: 1 } + - match: { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." } + + - do: + search: + index: test-sparse-index + body: + query: + semantic: + field: "body" + query: "What is Elasticsearch?" + highlight: + fields: + body: + type: "semantic" + number_of_fragments: 2 + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "doc_1" } + - length: { hits.hits.0.highlight.body: 2 } + - match: { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." } + - match: { hits.hits.0.highlight.body.1: "You Know, for Search!" } + + - do: + search: + index: test-sparse-index + body: + query: + semantic: + field: "body" + query: "What is Elasticsearch?" + highlight: + fields: + body: + type: "semantic" + order: "score" + number_of_fragments: 1 + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "doc_1" } + - length: { hits.hits.0.highlight.body: 1 } + - match: { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." } + + - do: + search: + index: test-sparse-index + body: + query: + semantic: + field: "body" + query: "What is Elasticsearch?" + highlight: + fields: + body: + type: "semantic" + order: "score" + number_of_fragments: 2 + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "doc_1" } + - length: { hits.hits.0.highlight.body: 2 } + - match: { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." } + - match: { hits.hits.0.highlight.body.1: "You Know, for Search!" } + +--- +"Highlighting using a dense embedding model": + - do: + index: + index: test-dense-index + id: doc_1 + body: + body: ["ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides.", "You Know, for Search!"] + refresh: true + + - match: { result: created } + + - do: + search: + index: test-dense-index + body: + query: + semantic: + field: "body" + query: "What is Elasticsearch?" + highlight: + fields: + body: + type: "semantic" + number_of_fragments: 1 + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "doc_1" } + - length: { hits.hits.0.highlight.body: 1 } + - match: { hits.hits.0.highlight.body.0: "You Know, for Search!" } + + - do: + search: + index: test-dense-index + body: + query: + semantic: + field: "body" + query: "What is Elasticsearch?" + highlight: + fields: + body: + type: "semantic" + number_of_fragments: 2 + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "doc_1" } + - length: { hits.hits.0.highlight.body: 2 } + - match: { hits.hits.0.highlight.body.0: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." } + - match: { hits.hits.0.highlight.body.1: "You Know, for Search!" } + + - do: + search: + index: test-dense-index + body: + query: + semantic: + field: "body" + query: "What is Elasticsearch?" + highlight: + fields: + body: + type: "semantic" + order: "score" + number_of_fragments: 1 + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "doc_1" } + - length: { hits.hits.0.highlight.body: 1 } + - match: { hits.hits.0.highlight.body.0: "You Know, for Search!" } + + - do: + search: + index: test-dense-index + body: + query: + semantic: + field: "body" + query: "What is Elasticsearch?" + highlight: + fields: + body: + type: "semantic" + order: "score" + number_of_fragments: 2 + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "doc_1" } + - length: { hits.hits.0.highlight.body: 2 } + - match: { hits.hits.0.highlight.body.0: "You Know, for Search!" } + - match: { hits.hits.0.highlight.body.1: "ElasticSearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." } + + From a04d67180c552a549fbac644bacd209b4e0370a4 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 6 Dec 2024 12:01:36 -0800 Subject: [PATCH 091/204] Allow early termination of exchange source (#118129) This change introduces the ability to gracefully terminate the exchange source early by instructing all remote exchange sinks to stop their computations. 1. When sufficient data has been accumulated (e.g., reaching the LIMIT), the exchange source signals remote sinks to stop generating new pages, allowing the query to finish sooner. 2. When users request immediate results, even if they are partial, incomplete, or potentially inaccurate. --- .../operator/exchange/ExchangeService.java | 31 ++++++++--- .../exchange/ExchangeSourceHandler.java | 32 ++++++++++- .../compute/operator/exchange/RemoteSink.java | 9 +-- .../exchange/ExchangeServiceTests.java | 55 ++++++++++++++++++- 4 files changed, 111 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java index 00c68c4f48e86..62cc4daf5fde5 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ChannelActionListener; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -23,6 +24,7 @@ import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockStreamInput; +import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.CancellableTask; @@ -40,10 +42,11 @@ import java.io.IOException; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; /** * {@link ExchangeService} is responsible for exchanging pages between exchange sinks and sources on the same or different nodes. @@ -293,7 +296,7 @@ static final class TransportRemoteSink implements RemoteSink { final Executor responseExecutor; final AtomicLong estimatedPageSizeInBytes = new AtomicLong(0L); - final AtomicBoolean finished = new AtomicBoolean(false); + final AtomicReference> completionListenerRef = new AtomicReference<>(null); TransportRemoteSink( TransportService transportService, @@ -318,13 +321,14 @@ public void fetchPageAsync(boolean allSourcesFinished, ActionListener completionListener = completionListenerRef.get(); + if (completionListener != null) { + completionListener.addListener(listener.map(unused -> new ExchangeResponse(blockFactory, null, true))); return; } doFetchPageAsync(false, ActionListener.wrap(r -> { if (r.finished()) { - finished.set(true); + completionListenerRef.compareAndSet(null, SubscribableListener.newSucceeded(null)); } listener.onResponse(r); }, e -> close(ActionListener.running(() -> listener.onFailure(e))))); @@ -356,10 +360,19 @@ private void doFetchPageAsync(boolean allSourcesFinished, ActionListener listener) { - if (finished.compareAndSet(false, true)) { - doFetchPageAsync(true, listener.delegateFailure((l, unused) -> l.onResponse(null))); - } else { - listener.onResponse(null); + final SubscribableListener candidate = new SubscribableListener<>(); + final SubscribableListener actual = completionListenerRef.updateAndGet( + curr -> Objects.requireNonNullElse(curr, candidate) + ); + actual.addListener(listener); + if (candidate == actual) { + doFetchPageAsync(true, ActionListener.wrap(r -> { + final Page page = r.takePage(); + if (page != null) { + page.releaseBlocks(); + } + candidate.onResponse(null); + }, e -> candidate.onResponse(null))); } } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java index b53ddea3da587..aa722695b841e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.compute.EsqlRefCountingListener; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.FailureCollector; @@ -19,6 +20,7 @@ import org.elasticsearch.core.Releasable; import java.util.List; +import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; @@ -41,6 +43,9 @@ public final class ExchangeSourceHandler { // The final failure collected will be notified to callers via the {@code completionListener}. private final FailureCollector failure = new FailureCollector(); + private final AtomicInteger nextSinkId = new AtomicInteger(); + private final Map remoteSinks = ConcurrentCollections.newConcurrentMap(); + /** * Creates a new ExchangeSourceHandler. * @@ -53,7 +58,9 @@ public ExchangeSourceHandler(int maxBufferSize, Executor fetchExecutor, ActionLi this.buffer = new ExchangeBuffer(maxBufferSize); this.fetchExecutor = fetchExecutor; this.outstandingSinks = new PendingInstances(() -> buffer.finish(false)); - this.outstandingSources = new PendingInstances(() -> buffer.finish(true)); + final PendingInstances closingSinks = new PendingInstances(() -> {}); + closingSinks.trackNewInstance(); + this.outstandingSources = new PendingInstances(() -> finishEarly(true, ActionListener.running(closingSinks::finishInstance))); buffer.addCompletionListener(ActionListener.running(() -> { final ActionListener listener = ActionListener.assertAtLeastOnce(completionListener); try (RefCountingRunnable refs = new RefCountingRunnable(() -> { @@ -64,6 +71,7 @@ public ExchangeSourceHandler(int maxBufferSize, Executor fetchExecutor, ActionLi listener.onResponse(null); } })) { + closingSinks.completion.addListener(refs.acquireListener()); for (PendingInstances pending : List.of(outstandingSinks, outstandingSources)) { // Create an outstanding instance and then finish to complete the completionListener // if we haven't registered any instances of exchange sinks or exchange sources before. @@ -257,7 +265,11 @@ void onSinkComplete() { * @see ExchangeSinkHandler#fetchPageAsync(boolean, ActionListener) */ public void addRemoteSink(RemoteSink remoteSink, boolean failFast, int instances, ActionListener listener) { - final ActionListener sinkListener = ActionListener.assertAtLeastOnce(ActionListener.notifyOnce(listener)); + final int sinkId = nextSinkId.incrementAndGet(); + remoteSinks.put(sinkId, remoteSink); + final ActionListener sinkListener = ActionListener.assertAtLeastOnce( + ActionListener.notifyOnce(ActionListener.runBefore(listener, () -> remoteSinks.remove(sinkId))) + ); fetchExecutor.execute(new AbstractRunnable() { @Override public void onFailure(Exception e) { @@ -291,6 +303,22 @@ public Releasable addEmptySink() { return outstandingSinks::finishInstance; } + /** + * Gracefully terminates the exchange source early by instructing all remote exchange sinks to stop their computations. + * This can happen when the exchange source has accumulated enough data (e.g., reaching the LIMIT) or when users want to + * see the current result immediately. + * + * @param drainingPages whether to discard pages already fetched in the exchange + */ + public void finishEarly(boolean drainingPages, ActionListener listener) { + buffer.finish(drainingPages); + try (EsqlRefCountingListener refs = new EsqlRefCountingListener(listener)) { + for (RemoteSink remoteSink : remoteSinks.values()) { + remoteSink.close(refs.acquire()); + } + } + } + private static class PendingInstances { private final AtomicInteger instances = new AtomicInteger(); private final SubscribableListener completion = new SubscribableListener<>(); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/RemoteSink.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/RemoteSink.java index aaa937ef17c0e..63b5d324ce851 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/RemoteSink.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/RemoteSink.java @@ -8,6 +8,7 @@ package org.elasticsearch.compute.operator.exchange; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.compute.data.Page; public interface RemoteSink { @@ -15,11 +16,11 @@ public interface RemoteSink { default void close(ActionListener listener) { fetchPageAsync(true, listener.delegateFailure((l, r) -> { - try { - r.close(); - } finally { - l.onResponse(null); + final Page page = r.takePage(); + if (page != null) { + page.releaseBlocks(); } + l.onResponse(null); })); } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java index fc6c850ba187b..8f7532b582bc2 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java @@ -55,7 +55,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Queue; import java.util.Set; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -421,7 +423,7 @@ public void testExchangeSourceContinueOnFailure() { } } - public void testEarlyTerminate() { + public void testClosingSinks() { BlockFactory blockFactory = blockFactory(); IntBlock block1 = blockFactory.newConstantIntBlockWith(1, 2); IntBlock block2 = blockFactory.newConstantIntBlockWith(1, 2); @@ -441,6 +443,57 @@ public void testEarlyTerminate() { assertTrue(sink.isFinished()); } + public void testFinishEarly() throws Exception { + ExchangeSourceHandler sourceHandler = new ExchangeSourceHandler(20, threadPool.generic(), ActionListener.noop()); + Semaphore permits = new Semaphore(between(1, 5)); + BlockFactory blockFactory = blockFactory(); + Queue pages = ConcurrentCollections.newQueue(); + ExchangeSource exchangeSource = sourceHandler.createExchangeSource(); + AtomicBoolean sinkClosed = new AtomicBoolean(); + PlainActionFuture sinkCompleted = new PlainActionFuture<>(); + sourceHandler.addRemoteSink((allSourcesFinished, listener) -> { + if (allSourcesFinished) { + sinkClosed.set(true); + permits.release(10); + listener.onResponse(new ExchangeResponse(blockFactory, null, sinkClosed.get())); + } else { + try { + if (permits.tryAcquire(between(0, 100), TimeUnit.MICROSECONDS)) { + boolean closed = sinkClosed.get(); + final Page page; + if (closed) { + page = new Page(blockFactory.newConstantIntBlockWith(1, 1)); + pages.add(page); + } else { + page = null; + } + listener.onResponse(new ExchangeResponse(blockFactory, page, closed)); + } else { + listener.onResponse(new ExchangeResponse(blockFactory, null, sinkClosed.get())); + } + } catch (Exception e) { + throw new AssertionError(e); + } + } + }, false, between(1, 3), sinkCompleted); + threadPool.schedule( + () -> sourceHandler.finishEarly(randomBoolean(), ActionListener.noop()), + TimeValue.timeValueMillis(between(0, 10)), + threadPool.generic() + ); + sinkCompleted.actionGet(); + Page p; + while ((p = exchangeSource.pollPage()) != null) { + assertSame(p, pages.poll()); + p.releaseBlocks(); + } + while ((p = pages.poll()) != null) { + p.releaseBlocks(); + } + assertTrue(exchangeSource.isFinished()); + exchangeSource.finish(); + } + public void testConcurrentWithTransportActions() { MockTransportService node0 = newTransportService(); ExchangeService exchange0 = new ExchangeService(Settings.EMPTY, threadPool, ESQL_TEST_EXECUTOR, blockFactory()); From 11061713c51552fa04c16bdc74785454350159a4 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 6 Dec 2024 21:17:50 +0100 Subject: [PATCH 092/204] Unmute #108628 (#118152) This should be fixed for a while now. Closes #116249 --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 887b462fa122e..07072e9743c98 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -117,9 +117,6 @@ tests: - class: org.elasticsearch.xpack.deprecation.DeprecationHttpIT method: testDeprecatedSettingsReturnWarnings issue: https://github.com/elastic/elasticsearch/issues/108628 -- class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests - method: testBottomFieldSort - issue: https://github.com/elastic/elasticsearch/issues/116249 - class: org.elasticsearch.xpack.shutdown.NodeShutdownIT method: testAllocationPreventedForRemoval issue: https://github.com/elastic/elasticsearch/issues/116363 From 2619149bc677fec18376fb36c5a54aa3d260431d Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 6 Dec 2024 21:41:27 +0100 Subject: [PATCH 093/204] [TEST] Work around race condition when starting clusters in parallel (#118145) With #117820 we are starting clusters in parallel in CCS related tests. There is a race condition when calling RandomizedTest#isNightly, which is called as part of `InternalTestCluster#beforeTest`. See https://github.com/randomizedtesting/randomizedtesting/issues/311. Until this is fixed upstream, we can simply call isNightly before forking, which is going to trigger the lazy initialization of the inner map that causes the race. From then on, all threads will have a consistent view of it. Closes #118124 --- muted-tests.yml | 3 --- .../elasticsearch/test/AbstractMultiClustersTestCase.java | 7 +++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 07072e9743c98..a39265756599d 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -239,9 +239,6 @@ tests: - class: org.elasticsearch.packaging.test.ConfigurationTests method: test30SymlinkedDataPath issue: https://github.com/elastic/elasticsearch/issues/118111 -- class: org.elasticsearch.datastreams.ResolveClusterDataStreamIT - method: testClusterResolveWithDataStreamsUsingAlias - issue: https://github.com/elastic/elasticsearch/issues/118124 - class: org.elasticsearch.packaging.test.KeystoreManagementTests method: test30KeystorePasswordFromFile issue: https://github.com/elastic/elasticsearch/issues/118123 diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java index b4f91f68b8bb7..7cd7bce4db187 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java @@ -9,6 +9,8 @@ package org.elasticsearch.test; +import com.carrotsearch.randomizedtesting.RandomizedTest; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.cluster.remote.RemoteInfoRequest; @@ -108,6 +110,11 @@ public final void startClusters() throws Exception { MockTransportService.TestPlugin.class, getTestTransportPlugin() ); + // We are going to initialize multiple clusters concurrently, but there is a race condition around the lazy initialization of test + // groups in GroupEvaluator across multiple threads. See https://github.com/randomizedtesting/randomizedtesting/issues/311. + // Calling isNightly before parallelizing is enough to work around that issue. + @SuppressWarnings("unused") + boolean nightly = RandomizedTest.isNightly(); runInParallel(clusterAliases.size(), i -> { String clusterAlias = clusterAliases.get(i); final String clusterName = clusterAlias.equals(LOCAL_CLUSTER) ? "main-cluster" : clusterAlias; From 467fdb879c6ed1ea084d02087a4afaa12babe7c6 Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Fri, 6 Dec 2024 15:52:31 -0500 Subject: [PATCH 094/204] [Inference API] Add unified api for chat completions (#117589) * Adding some shell classes * modeling the request objects * Writeable changes to schema * Working parsing tests * Creating a new action * Add outbound request writing (WIP) * Improvements to request serialization * Adding separate transport classes * separate out unified request and combine inputs * Reworking unified inputs * Adding unsupported operation calls * Fixing parsing logic * get the build working * Update docs/changelog/117589.yaml * Fixing injection issue * Allowing model to be overridden but not working yet * Fixing issues * Switch field name for tool * Add suport for toolCalls and refusal in streaming completion * Working tool call response * Separate unified and legacy code paths * Updated the parser, but there are some class cast exceptions to fix * Refactoring tests and request entities * Parse response from OpenAI * Removing unused request classes * precommit * Adding tests for UnifiedCompletionAction Request * Refactoring stop to be a list of strings * Testing for OpenAI response parsing * Refactoring transport action tests to test unified validation code * Fixing various tests * Fixing license header * Reformat streaming results * Finalize response format * remove debug logs * remove changes for debugging * Task type and base inference action tests * Adding openai service tests * Adding model tests * tests for StreamingUnifiedChatCompletionResultsTests toXContentChunked * Fixing change log and removing commented out code * Switch usage to accept null * Adding test for TestStreamingCompletionServiceExtension * Avoid serializing empty lists + request entity tests * Register named writeables from UnifiedCompletionRequest * Removing commented code * Clean up and add more of an explination * remove duplicate test * remove old todos * Refactoring some duplication * Adding javadoc * Addressing feedback --------- Co-authored-by: Jonathan Buttner Co-authored-by: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> --- docs/changelog/117589.yaml | 5 + .../xcontent/ChunkedToXContentHelper.java | 9 + .../inference/InferenceService.java | 17 + .../org/elasticsearch/inference/TaskType.java | 4 + .../inference/UnifiedCompletionRequest.java | 425 +++++++++ .../org/elasticsearch/test/ESTestCase.java | 22 +- .../action/BaseInferenceActionRequest.java | 31 + .../inference/action/InferenceAction.java | 3 +- .../action/UnifiedCompletionAction.java | 129 +++ ...StreamingUnifiedChatCompletionResults.java | 329 +++++++ .../action/InferenceActionRequestTests.java | 3 +- .../UnifiedCompletionActionRequestTests.java | 97 ++ .../action/UnifiedCompletionRequestTests.java | 293 ++++++ ...mingUnifiedChatCompletionResultsTests.java | 198 ++++ .../authz/store/ReservedRolesStoreTests.java | 1 + .../inference/InferenceBaseRestTest.java | 32 +- .../xpack/inference/InferenceCrudIT.java | 55 ++ .../TestDenseInferenceServiceExtension.java | 11 + .../mock/TestRerankingServiceExtension.java | 11 + .../TestSparseInferenceServiceExtension.java | 11 + ...stStreamingCompletionServiceExtension.java | 73 ++ .../InferenceNamedWriteablesProvider.java | 8 + .../xpack/inference/InferencePlugin.java | 20 +- .../inference/UnifiedCompletionFeature.java | 20 + .../action/BaseTransportInferenceAction.java | 250 +++++ .../action/TransportInferenceAction.java | 219 +---- ...sportUnifiedCompletionInferenceAction.java | 77 ++ .../inference/common/DelegatingProcessor.java | 45 +- .../SingleInputSenderExecutableAction.java | 9 +- .../action/openai/OpenAiActionCreator.java | 2 +- ...baCloudSearchCompletionRequestManager.java | 2 +- ...onBedrockChatCompletionRequestManager.java | 8 +- .../AnthropicCompletionRequestManager.java | 8 +- ...eAiStudioChatCompletionRequestManager.java | 8 +- .../AzureOpenAiCompletionRequestManager.java | 8 +- .../http/sender/ChatCompletionInput.java | 39 + .../CohereCompletionRequestManager.java | 8 +- .../http/sender/DocumentsOnlyInput.java | 10 +- ...oogleAiStudioCompletionRequestManager.java | 5 +- .../external/http/sender/InferenceInputs.java | 26 +- .../OpenAiCompletionRequestManager.java | 12 +- ...OpenAiUnifiedCompletionRequestManager.java | 61 ++ .../http/sender/QueryAndDocsInputs.java | 11 +- .../http/sender/UnifiedChatInput.java | 62 ++ .../openai/OpenAiStreamingProcessor.java | 18 +- ...iUnifiedChatCompletionResponseHandler.java | 34 + .../OpenAiUnifiedStreamingProcessor.java | 287 ++++++ .../GoogleAiStudioCompletionRequest.java | 6 +- .../OpenAiChatCompletionRequestEntity.java | 79 -- ...> OpenAiUnifiedChatCompletionRequest.java} | 20 +- ...nAiUnifiedChatCompletionRequestEntity.java | 185 ++++ .../inference/rest/BaseInferenceAction.java | 32 +- .../xpack/inference/rest/Paths.java | 6 + .../RestUnifiedCompletionInferenceAction.java | 49 + .../inference/services/SenderService.java | 43 +- .../inference/services/ServiceUtils.java | 4 + .../AlibabaCloudSearchService.java | 14 +- .../amazonbedrock/AmazonBedrockService.java | 12 + .../services/anthropic/AnthropicService.java | 12 + .../azureaistudio/AzureAiStudioService.java | 12 + .../azureopenai/AzureOpenAiService.java | 12 + .../services/cohere/CohereService.java | 12 + .../elastic/ElasticInferenceService.java | 12 + .../ElasticsearchInternalService.java | 12 + .../googleaistudio/GoogleAiStudioService.java | 15 +- .../googlevertexai/GoogleVertexAiService.java | 12 + .../huggingface/HuggingFaceService.java | 13 + .../elser/HuggingFaceElserService.java | 12 + .../ibmwatsonx/IbmWatsonxService.java | 12 + .../services/mistral/MistralService.java | 12 + .../services/openai/OpenAiService.java | 27 + .../completion/OpenAiChatCompletionModel.java | 22 + ...enAiChatCompletionRequestTaskSettings.java | 1 - .../xpack/inference/TaskTypeTests.java | 27 + .../elasticsearch/xpack/inference/Utils.java | 3 + .../BaseTransportInferenceActionTestCase.java | 364 ++++++++ .../action/TransportInferenceActionTests.java | 337 +------ ...TransportUnifiedCompletionActionTests.java | 124 +++ ...ingleInputSenderExecutableActionTests.java | 20 +- .../AmazonBedrockActionCreatorTests.java | 5 +- .../AnthropicActionCreatorTests.java | 7 +- .../AnthropicChatCompletionActionTests.java | 12 +- .../AzureAiStudioActionAndCreatorTests.java | 3 +- .../AzureOpenAiActionCreatorTests.java | 7 +- .../AzureOpenAiCompletionActionTests.java | 10 +- .../cohere/CohereActionCreatorTests.java | 5 +- .../cohere/CohereCompletionActionTests.java | 18 +- .../GoogleAiStudioCompletionActionTests.java | 12 +- .../openai/OpenAiActionCreatorTests.java | 21 +- .../OpenAiChatCompletionActionTests.java | 19 +- .../AmazonBedrockMockRequestSender.java | 12 +- .../AmazonBedrockRequestSenderTests.java | 3 +- .../http/sender/InferenceInputsTests.java | 40 + .../http/sender/UnifiedChatInputTests.java | 46 + .../OpenAiUnifiedStreamingProcessorTests.java | 383 ++++++++ .../GoogleAiStudioCompletionRequestTests.java | 6 +- ...penAiChatCompletionRequestEntityTests.java | 53 -- ...ifiedChatCompletionRequestEntityTests.java | 856 ++++++++++++++++++ ...nAiUnifiedChatCompletionRequestTests.java} | 59 +- .../rest/BaseInferenceActionTests.java | 43 + ...UnifiedCompletionInferenceActionTests.java | 81 ++ .../services/SenderServiceTests.java | 9 + .../services/openai/OpenAiServiceTests.java | 63 ++ .../OpenAiChatCompletionModelTests.java | 42 +- .../xpack/security/operator/Constants.java | 1 + 105 files changed, 5488 insertions(+), 867 deletions(-) create mode 100644 docs/changelog/117589.yaml create mode 100644 server/src/main/java/org/elasticsearch/inference/UnifiedCompletionRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/BaseInferenceActionRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResults.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionActionRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResultsTests.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/UnifiedCompletionFeature.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ChatCompletionInput.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiUnifiedCompletionRequestManager.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInput.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandler.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessor.java delete mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntity.java rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/{OpenAiChatCompletionRequest.java => OpenAiUnifiedChatCompletionRequest.java} (80%) create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntity.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceAction.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/TaskTypeTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceInputsTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInputTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessorTests.java delete mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntityTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntityTests.java rename x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/{OpenAiChatCompletionRequestTests.java => OpenAiUnifiedChatCompletionRequestTests.java} (75%) create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceActionTests.java diff --git a/docs/changelog/117589.yaml b/docs/changelog/117589.yaml new file mode 100644 index 0000000000000..e6880fd9477b5 --- /dev/null +++ b/docs/changelog/117589.yaml @@ -0,0 +1,5 @@ +pr: 117589 +summary: "Add Inference Unified API for chat completions for OpenAI" +area: Machine Learning +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java index 2e78cc6f516b1..6a5aa2943de92 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.xcontent.ToXContent; +import java.util.Collections; import java.util.Iterator; public enum ChunkedToXContentHelper { @@ -53,6 +54,14 @@ public static Iterator field(String name, String value) { return Iterators.single(((builder, params) -> builder.field(name, value))); } + public static Iterator optionalField(String name, String value) { + if (value == null) { + return Collections.emptyIterator(); + } else { + return field(name, value); + } + } + /** * Creates an Iterator of a single ToXContent object that serializes the given object as a single chunk. Just wraps {@link * Iterators#single}, but still useful because it avoids any type ambiguity. diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceService.java b/server/src/main/java/org/elasticsearch/inference/InferenceService.java index 4497254aad1f0..c2d690d8160ac 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceService.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceService.java @@ -112,6 +112,23 @@ void infer( ); /** + * Perform completion inference on the model using the unified schema. + * + * @param model The model + * @param request Parameters for the request + * @param timeout The timeout for the request + * @param listener Inference result listener + */ + void unifiedCompletionInfer( + Model model, + UnifiedCompletionRequest request, + TimeValue timeout, + ActionListener listener + ); + + /** + * Chunk long text. + * * @param model The model * @param query Inference query, mainly for re-ranking * @param input Inference input diff --git a/server/src/main/java/org/elasticsearch/inference/TaskType.java b/server/src/main/java/org/elasticsearch/inference/TaskType.java index b0e5bababbbc0..fcb8ea7213795 100644 --- a/server/src/main/java/org/elasticsearch/inference/TaskType.java +++ b/server/src/main/java/org/elasticsearch/inference/TaskType.java @@ -38,6 +38,10 @@ public static TaskType fromString(String name) { } public static TaskType fromStringOrStatusException(String name) { + if (name == null) { + throw new ElasticsearchStatusException("Task type must not be null", RestStatus.BAD_REQUEST); + } + try { TaskType taskType = TaskType.fromString(name); return Objects.requireNonNull(taskType); diff --git a/server/src/main/java/org/elasticsearch/inference/UnifiedCompletionRequest.java b/server/src/main/java/org/elasticsearch/inference/UnifiedCompletionRequest.java new file mode 100644 index 0000000000000..e596be626b518 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/inference/UnifiedCompletionRequest.java @@ -0,0 +1,425 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.inference; + +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParseException; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public record UnifiedCompletionRequest( + List messages, + @Nullable String model, + @Nullable Long maxCompletionTokens, + @Nullable List stop, + @Nullable Float temperature, + @Nullable ToolChoice toolChoice, + @Nullable List tools, + @Nullable Float topP +) implements Writeable { + + public sealed interface Content extends NamedWriteable permits ContentObjects, ContentString {} + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + UnifiedCompletionRequest.class.getSimpleName(), + args -> new UnifiedCompletionRequest( + (List) args[0], + (String) args[1], + (Long) args[2], + (List) args[3], + (Float) args[4], + (ToolChoice) args[5], + (List) args[6], + (Float) args[7] + ) + ); + + static { + PARSER.declareObjectArray(constructorArg(), Message.PARSER::apply, new ParseField("messages")); + PARSER.declareString(optionalConstructorArg(), new ParseField("model")); + PARSER.declareLong(optionalConstructorArg(), new ParseField("max_completion_tokens")); + PARSER.declareStringArray(optionalConstructorArg(), new ParseField("stop")); + PARSER.declareFloat(optionalConstructorArg(), new ParseField("temperature")); + PARSER.declareField( + optionalConstructorArg(), + (p, c) -> parseToolChoice(p), + new ParseField("tool_choice"), + ObjectParser.ValueType.OBJECT_OR_STRING + ); + PARSER.declareObjectArray(optionalConstructorArg(), Tool.PARSER::apply, new ParseField("tools")); + PARSER.declareFloat(optionalConstructorArg(), new ParseField("top_p")); + } + + public static List getNamedWriteables() { + return List.of( + new NamedWriteableRegistry.Entry(Content.class, ContentObjects.NAME, ContentObjects::new), + new NamedWriteableRegistry.Entry(Content.class, ContentString.NAME, ContentString::new), + new NamedWriteableRegistry.Entry(ToolChoice.class, ToolChoiceObject.NAME, ToolChoiceObject::new), + new NamedWriteableRegistry.Entry(ToolChoice.class, ToolChoiceString.NAME, ToolChoiceString::new) + ); + } + + public static UnifiedCompletionRequest of(List messages) { + return new UnifiedCompletionRequest(messages, null, null, null, null, null, null, null); + } + + public UnifiedCompletionRequest(StreamInput in) throws IOException { + this( + in.readCollectionAsImmutableList(Message::new), + in.readOptionalString(), + in.readOptionalVLong(), + in.readOptionalStringCollectionAsList(), + in.readOptionalFloat(), + in.readOptionalNamedWriteable(ToolChoice.class), + in.readOptionalCollectionAsList(Tool::new), + in.readOptionalFloat() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(messages); + out.writeOptionalString(model); + out.writeOptionalVLong(maxCompletionTokens); + out.writeOptionalStringCollection(stop); + out.writeOptionalFloat(temperature); + out.writeOptionalNamedWriteable(toolChoice); + out.writeOptionalCollection(tools); + out.writeOptionalFloat(topP); + } + + public record Message(Content content, String role, @Nullable String name, @Nullable String toolCallId, List toolCalls) + implements + Writeable { + + @SuppressWarnings("unchecked") + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + Message.class.getSimpleName(), + args -> new Message((Content) args[0], (String) args[1], (String) args[2], (String) args[3], (List) args[4]) + ); + + static { + PARSER.declareField(constructorArg(), (p, c) -> parseContent(p), new ParseField("content"), ObjectParser.ValueType.VALUE_ARRAY); + PARSER.declareString(constructorArg(), new ParseField("role")); + PARSER.declareString(optionalConstructorArg(), new ParseField("name")); + PARSER.declareString(optionalConstructorArg(), new ParseField("tool_call_id")); + PARSER.declareObjectArray(optionalConstructorArg(), ToolCall.PARSER::apply, new ParseField("tool_calls")); + } + + private static Content parseContent(XContentParser parser) throws IOException { + var token = parser.currentToken(); + if (token == XContentParser.Token.START_ARRAY) { + var parsedContentObjects = XContentParserUtils.parseList(parser, (p) -> ContentObject.PARSER.apply(p, null)); + return new ContentObjects(parsedContentObjects); + } else if (token == XContentParser.Token.VALUE_STRING) { + return ContentString.of(parser); + } + + throw new XContentParseException("Expected an array start token or a value string token but found token [" + token + "]"); + } + + public Message(StreamInput in) throws IOException { + this( + in.readNamedWriteable(Content.class), + in.readString(), + in.readOptionalString(), + in.readOptionalString(), + in.readOptionalCollectionAsList(ToolCall::new) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(content); + out.writeString(role); + out.writeOptionalString(name); + out.writeOptionalString(toolCallId); + out.writeOptionalCollection(toolCalls); + } + } + + public record ContentObjects(List contentObjects) implements Content, NamedWriteable { + + public static final String NAME = "content_objects"; + + public ContentObjects(StreamInput in) throws IOException { + this(in.readCollectionAsImmutableList(ContentObject::new)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(contentObjects); + } + + @Override + public String getWriteableName() { + return NAME; + } + } + + public record ContentObject(String text, String type) implements Writeable { + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + ContentObject.class.getSimpleName(), + args -> new ContentObject((String) args[0], (String) args[1]) + ); + + static { + PARSER.declareString(constructorArg(), new ParseField("text")); + PARSER.declareString(constructorArg(), new ParseField("type")); + } + + public ContentObject(StreamInput in) throws IOException { + this(in.readString(), in.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(text); + out.writeString(type); + } + + public String toString() { + return text + ":" + type; + } + + } + + public record ContentString(String content) implements Content, NamedWriteable { + public static final String NAME = "content_string"; + + public static ContentString of(XContentParser parser) throws IOException { + var content = parser.text(); + return new ContentString(content); + } + + public ContentString(StreamInput in) throws IOException { + this(in.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(content); + } + + @Override + public String getWriteableName() { + return NAME; + } + + public String toString() { + return content; + } + } + + public record ToolCall(String id, FunctionField function, String type) implements Writeable { + + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + ToolCall.class.getSimpleName(), + args -> new ToolCall((String) args[0], (FunctionField) args[1], (String) args[2]) + ); + + static { + PARSER.declareString(constructorArg(), new ParseField("id")); + PARSER.declareObject(constructorArg(), FunctionField.PARSER::apply, new ParseField("function")); + PARSER.declareString(constructorArg(), new ParseField("type")); + } + + public ToolCall(StreamInput in) throws IOException { + this(in.readString(), new FunctionField(in), in.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + function.writeTo(out); + out.writeString(type); + } + + public record FunctionField(String arguments, String name) implements Writeable { + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "tool_call_function_field", + args -> new FunctionField((String) args[0], (String) args[1]) + ); + + static { + PARSER.declareString(constructorArg(), new ParseField("arguments")); + PARSER.declareString(constructorArg(), new ParseField("name")); + } + + public FunctionField(StreamInput in) throws IOException { + this(in.readString(), in.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(arguments); + out.writeString(name); + } + } + } + + private static ToolChoice parseToolChoice(XContentParser parser) throws IOException { + var token = parser.currentToken(); + if (token == XContentParser.Token.START_OBJECT) { + return ToolChoiceObject.PARSER.apply(parser, null); + } else if (token == XContentParser.Token.VALUE_STRING) { + return ToolChoiceString.of(parser); + } + + throw new XContentParseException("Unsupported token [" + token + "]"); + } + + public sealed interface ToolChoice extends NamedWriteable permits ToolChoiceObject, ToolChoiceString {} + + public record ToolChoiceObject(String type, FunctionField function) implements ToolChoice, NamedWriteable { + + public static final String NAME = "tool_choice_object"; + + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + ToolChoiceObject.class.getSimpleName(), + args -> new ToolChoiceObject((String) args[0], (FunctionField) args[1]) + ); + + static { + PARSER.declareString(constructorArg(), new ParseField("type")); + PARSER.declareObject(constructorArg(), FunctionField.PARSER::apply, new ParseField("function")); + } + + public ToolChoiceObject(StreamInput in) throws IOException { + this(in.readString(), new FunctionField(in)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(type); + function.writeTo(out); + } + + @Override + public String getWriteableName() { + return NAME; + } + + public record FunctionField(String name) implements Writeable { + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "tool_choice_function_field", + args -> new FunctionField((String) args[0]) + ); + + static { + PARSER.declareString(constructorArg(), new ParseField("name")); + } + + public FunctionField(StreamInput in) throws IOException { + this(in.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + } + } + } + + public record ToolChoiceString(String value) implements ToolChoice, NamedWriteable { + public static final String NAME = "tool_choice_string"; + + public static ToolChoiceString of(XContentParser parser) throws IOException { + var content = parser.text(); + return new ToolChoiceString(content); + } + + public ToolChoiceString(StreamInput in) throws IOException { + this(in.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(value); + } + + @Override + public String getWriteableName() { + return NAME; + } + } + + public record Tool(String type, FunctionField function) implements Writeable { + + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + Tool.class.getSimpleName(), + args -> new Tool((String) args[0], (FunctionField) args[1]) + ); + + static { + PARSER.declareString(constructorArg(), new ParseField("type")); + PARSER.declareObject(constructorArg(), FunctionField.PARSER::apply, new ParseField("function")); + } + + public Tool(StreamInput in) throws IOException { + this(in.readString(), new FunctionField(in)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(type); + function.writeTo(out); + } + + public record FunctionField( + @Nullable String description, + String name, + @Nullable Map parameters, + @Nullable Boolean strict + ) implements Writeable { + + @SuppressWarnings("unchecked") + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "tool_function_field", + args -> new FunctionField((String) args[0], (String) args[1], (Map) args[2], (Boolean) args[3]) + ); + + static { + PARSER.declareString(optionalConstructorArg(), new ParseField("description")); + PARSER.declareString(constructorArg(), new ParseField("name")); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.mapOrdered(), new ParseField("parameters")); + PARSER.declareBoolean(optionalConstructorArg(), new ParseField("strict")); + } + + public FunctionField(StreamInput in) throws IOException { + this(in.readOptionalString(), in.readString(), in.readGenericMap(), in.readOptionalBoolean()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(description); + out.writeString(name); + out.writeGenericMap(parameters); + out.writeOptionalBoolean(strict); + } + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index d983fc854bdfd..a71f61740e17b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -1205,10 +1205,30 @@ public static SecureString randomSecureStringOfLength(int codeUnits) { return new SecureString(randomAlpha.toCharArray()); } - public static String randomNullOrAlphaOfLength(int codeUnits) { + public static String randomAlphaOfLengthOrNull(int codeUnits) { return randomBoolean() ? null : randomAlphaOfLength(codeUnits); } + public static Long randomLongOrNull() { + return randomBoolean() ? null : randomLong(); + } + + public static Long randomPositiveLongOrNull() { + return randomBoolean() ? null : randomNonNegativeLong(); + } + + public static Integer randomIntOrNull() { + return randomBoolean() ? null : randomInt(); + } + + public static Integer randomPositiveIntOrNull() { + return randomBoolean() ? null : randomNonNegativeInt(); + } + + public static Float randomFloatOrNull() { + return randomBoolean() ? null : randomFloat(); + } + /** * Creates a valid random identifier such as node id or index name */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/BaseInferenceActionRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/BaseInferenceActionRequest.java new file mode 100644 index 0000000000000..e426574c52ce6 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/BaseInferenceActionRequest.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.inference.TaskType; + +import java.io.IOException; + +public abstract class BaseInferenceActionRequest extends ActionRequest { + + public BaseInferenceActionRequest() { + super(); + } + + public BaseInferenceActionRequest(StreamInput in) throws IOException { + super(in); + } + + public abstract boolean isStreaming(); + + public abstract TaskType getTaskType(); + + public abstract String getInferenceEntityId(); +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java index a19edd5a08162..f88909ba4208e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java @@ -10,7 +10,6 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; -import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; @@ -54,7 +53,7 @@ public InferenceAction() { super(NAME); } - public static class Request extends ActionRequest { + public static class Request extends BaseInferenceActionRequest { public static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueSeconds(30); public static final ParseField INPUT = new ParseField("input"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionAction.java new file mode 100644 index 0000000000000..8d121463fb465 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionAction.java @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnifiedCompletionRequest; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +public class UnifiedCompletionAction extends ActionType { + public static final UnifiedCompletionAction INSTANCE = new UnifiedCompletionAction(); + public static final String NAME = "cluster:monitor/xpack/inference/unified"; + + public UnifiedCompletionAction() { + super(NAME); + } + + public static class Request extends BaseInferenceActionRequest { + public static Request parseRequest(String inferenceEntityId, TaskType taskType, TimeValue timeout, XContentParser parser) + throws IOException { + var unifiedRequest = UnifiedCompletionRequest.PARSER.apply(parser, null); + return new Request(inferenceEntityId, taskType, unifiedRequest, timeout); + } + + private final String inferenceEntityId; + private final TaskType taskType; + private final UnifiedCompletionRequest unifiedCompletionRequest; + private final TimeValue timeout; + + public Request(String inferenceEntityId, TaskType taskType, UnifiedCompletionRequest unifiedCompletionRequest, TimeValue timeout) { + this.inferenceEntityId = Objects.requireNonNull(inferenceEntityId); + this.taskType = Objects.requireNonNull(taskType); + this.unifiedCompletionRequest = Objects.requireNonNull(unifiedCompletionRequest); + this.timeout = Objects.requireNonNull(timeout); + } + + public Request(StreamInput in) throws IOException { + super(in); + this.inferenceEntityId = in.readString(); + this.taskType = TaskType.fromStream(in); + this.unifiedCompletionRequest = new UnifiedCompletionRequest(in); + this.timeout = in.readTimeValue(); + } + + public TaskType getTaskType() { + return taskType; + } + + public String getInferenceEntityId() { + return inferenceEntityId; + } + + public UnifiedCompletionRequest getUnifiedCompletionRequest() { + return unifiedCompletionRequest; + } + + /** + * The Unified API only supports streaming so we always return true here. + * @return true + */ + public boolean isStreaming() { + return true; + } + + public TimeValue getTimeout() { + return timeout; + } + + @Override + public ActionRequestValidationException validate() { + if (unifiedCompletionRequest == null || unifiedCompletionRequest.messages() == null) { + var e = new ActionRequestValidationException(); + e.addValidationError("Field [messages] cannot be null"); + return e; + } + + if (unifiedCompletionRequest.messages().isEmpty()) { + var e = new ActionRequestValidationException(); + e.addValidationError("Field [messages] cannot be an empty array"); + return e; + } + + if (taskType.isAnyOrSame(TaskType.COMPLETION) == false) { + var e = new ActionRequestValidationException(); + e.addValidationError("Field [taskType] must be [completion]"); + return e; + } + + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(inferenceEntityId); + taskType.writeTo(out); + unifiedCompletionRequest.writeTo(out); + out.writeTimeValue(timeout); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(inferenceEntityId, request.inferenceEntityId) + && taskType == request.taskType + && Objects.equals(unifiedCompletionRequest, request.unifiedCompletionRequest) + && Objects.equals(timeout, request.timeout); + } + + @Override + public int hashCode() { + return Objects.hash(inferenceEntityId, taskType, unifiedCompletionRequest, timeout); + } + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResults.java new file mode 100644 index 0000000000000..90038c67036c4 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResults.java @@ -0,0 +1,329 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference.results; + +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.inference.InferenceResults; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xcontent.ToXContent; + +import java.io.IOException; +import java.util.Collections; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Flow; + +/** + * Chat Completion results that only contain a Flow.Publisher. + */ +public record StreamingUnifiedChatCompletionResults(Flow.Publisher publisher) + implements + InferenceServiceResults { + + public static final String NAME = "chat_completion_chunk"; + public static final String MODEL_FIELD = "model"; + public static final String OBJECT_FIELD = "object"; + public static final String USAGE_FIELD = "usage"; + public static final String INDEX_FIELD = "index"; + public static final String ID_FIELD = "id"; + public static final String FUNCTION_NAME_FIELD = "name"; + public static final String FUNCTION_ARGUMENTS_FIELD = "arguments"; + public static final String FUNCTION_FIELD = "function"; + public static final String CHOICES_FIELD = "choices"; + public static final String DELTA_FIELD = "delta"; + public static final String CONTENT_FIELD = "content"; + public static final String REFUSAL_FIELD = "refusal"; + public static final String ROLE_FIELD = "role"; + private static final String TOOL_CALLS_FIELD = "tool_calls"; + public static final String FINISH_REASON_FIELD = "finish_reason"; + public static final String COMPLETION_TOKENS_FIELD = "completion_tokens"; + public static final String TOTAL_TOKENS_FIELD = "total_tokens"; + public static final String PROMPT_TOKENS_FIELD = "prompt_tokens"; + public static final String TYPE_FIELD = "type"; + + @Override + public boolean isStreaming() { + return true; + } + + @Override + public List transformToCoordinationFormat() { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public List transformToLegacyFormat() { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public Map asMap() { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public String getWriteableName() { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + throw new UnsupportedOperationException("Not implemented"); + } + + public record Results(Deque chunks) implements ChunkedToXContent { + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + return Iterators.concat(Iterators.flatMap(chunks.iterator(), c -> c.toXContentChunked(params))); + } + } + + public static class ChatCompletionChunk implements ChunkedToXContent { + private final String id; + + public String getId() { + return id; + } + + public List getChoices() { + return choices; + } + + public String getModel() { + return model; + } + + public String getObject() { + return object; + } + + public Usage getUsage() { + return usage; + } + + private final List choices; + private final String model; + private final String object; + private final ChatCompletionChunk.Usage usage; + + public ChatCompletionChunk(String id, List choices, String model, String object, ChatCompletionChunk.Usage usage) { + this.id = id; + this.choices = choices; + this.model = model; + this.object = object; + this.usage = usage; + } + + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + + Iterator choicesIterator = Collections.emptyIterator(); + if (choices != null) { + choicesIterator = Iterators.concat( + ChunkedToXContentHelper.startArray(CHOICES_FIELD), + Iterators.flatMap(choices.iterator(), c -> c.toXContentChunked(params)), + ChunkedToXContentHelper.endArray() + ); + } + + Iterator usageIterator = Collections.emptyIterator(); + if (usage != null) { + usageIterator = Iterators.concat( + ChunkedToXContentHelper.startObject(USAGE_FIELD), + ChunkedToXContentHelper.field(COMPLETION_TOKENS_FIELD, usage.completionTokens()), + ChunkedToXContentHelper.field(PROMPT_TOKENS_FIELD, usage.promptTokens()), + ChunkedToXContentHelper.field(TOTAL_TOKENS_FIELD, usage.totalTokens()), + ChunkedToXContentHelper.endObject() + ); + } + + return Iterators.concat( + ChunkedToXContentHelper.startObject(), + ChunkedToXContentHelper.field(ID_FIELD, id), + choicesIterator, + ChunkedToXContentHelper.field(MODEL_FIELD, model), + ChunkedToXContentHelper.field(OBJECT_FIELD, object), + usageIterator, + ChunkedToXContentHelper.endObject() + ); + } + + public record Choice(ChatCompletionChunk.Choice.Delta delta, String finishReason, int index) { + + /* + choices: Array<{ + delta: { ... }; + finish_reason: string | null; + index: number; + }>; + */ + public Iterator toXContentChunked(ToXContent.Params params) { + return Iterators.concat( + ChunkedToXContentHelper.startObject(), + delta.toXContentChunked(params), + ChunkedToXContentHelper.optionalField(FINISH_REASON_FIELD, finishReason), + ChunkedToXContentHelper.field(INDEX_FIELD, index), + ChunkedToXContentHelper.endObject() + ); + } + + public static class Delta { + private final String content; + private final String refusal; + private final String role; + private List toolCalls; + + public Delta(String content, String refusal, String role, List toolCalls) { + this.content = content; + this.refusal = refusal; + this.role = role; + this.toolCalls = toolCalls; + } + + /* + delta: { + content?: string | null; + refusal?: string | null; + role?: 'system' | 'user' | 'assistant' | 'tool'; + tool_calls?: Array<{ ... }>; + }; + */ + public Iterator toXContentChunked(ToXContent.Params params) { + var xContent = Iterators.concat( + ChunkedToXContentHelper.startObject(DELTA_FIELD), + ChunkedToXContentHelper.optionalField(CONTENT_FIELD, content), + ChunkedToXContentHelper.optionalField(REFUSAL_FIELD, refusal), + ChunkedToXContentHelper.optionalField(ROLE_FIELD, role) + ); + + if (toolCalls != null && toolCalls.isEmpty() == false) { + xContent = Iterators.concat( + xContent, + ChunkedToXContentHelper.startArray(TOOL_CALLS_FIELD), + Iterators.flatMap(toolCalls.iterator(), t -> t.toXContentChunked(params)), + ChunkedToXContentHelper.endArray() + ); + } + xContent = Iterators.concat(xContent, ChunkedToXContentHelper.endObject()); + return xContent; + + } + + public String getContent() { + return content; + } + + public String getRefusal() { + return refusal; + } + + public String getRole() { + return role; + } + + public List getToolCalls() { + return toolCalls; + } + + public static class ToolCall { + private final int index; + private final String id; + public ChatCompletionChunk.Choice.Delta.ToolCall.Function function; + private final String type; + + public ToolCall(int index, String id, ChatCompletionChunk.Choice.Delta.ToolCall.Function function, String type) { + this.index = index; + this.id = id; + this.function = function; + this.type = type; + } + + public int getIndex() { + return index; + } + + public String getId() { + return id; + } + + public ChatCompletionChunk.Choice.Delta.ToolCall.Function getFunction() { + return function; + } + + public String getType() { + return type; + } + + /* + index: number; + id?: string; + function?: { + arguments?: string; + name?: string; + }; + type?: 'function'; + */ + public Iterator toXContentChunked(ToXContent.Params params) { + var content = Iterators.concat( + ChunkedToXContentHelper.startObject(), + ChunkedToXContentHelper.field(INDEX_FIELD, index), + ChunkedToXContentHelper.optionalField(ID_FIELD, id) + ); + + if (function != null) { + content = Iterators.concat( + content, + ChunkedToXContentHelper.startObject(FUNCTION_FIELD), + ChunkedToXContentHelper.optionalField(FUNCTION_ARGUMENTS_FIELD, function.getArguments()), + ChunkedToXContentHelper.optionalField(FUNCTION_NAME_FIELD, function.getName()), + ChunkedToXContentHelper.endObject() + ); + } + + content = Iterators.concat( + content, + ChunkedToXContentHelper.field(TYPE_FIELD, type), + ChunkedToXContentHelper.endObject() + ); + return content; + } + + public static class Function { + private final String arguments; + private final String name; + + public Function(String arguments, String name) { + this.arguments = arguments; + this.name = name; + } + + public String getArguments() { + return arguments; + } + + public String getName() { + return name; + } + } + } + } + } + + public record Usage(int completionTokens, int promptTokens, int totalTokens) {} + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/InferenceActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/InferenceActionRequestTests.java index a9ca5e6da8720..01c0ff88be222 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/InferenceActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/InferenceActionRequestTests.java @@ -41,8 +41,7 @@ protected InferenceAction.Request createTestInstance() { return new InferenceAction.Request( randomFrom(TaskType.values()), randomAlphaOfLength(6), - // null, - randomNullOrAlphaOfLength(10), + randomAlphaOfLengthOrNull(10), randomList(1, 5, () -> randomAlphaOfLength(8)), randomMap(0, 3, () -> new Tuple<>(randomAlphaOfLength(4), randomAlphaOfLength(4))), randomFrom(InputType.values()), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionActionRequestTests.java new file mode 100644 index 0000000000000..1872ac3caa230 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionActionRequestTests.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnifiedCompletionRequest; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.is; + +public class UnifiedCompletionActionRequestTests extends AbstractBWCWireSerializationTestCase { + + public void testValidation_ReturnsException_When_UnifiedCompletionRequestMessage_Is_Null() { + var request = new UnifiedCompletionAction.Request( + "inference_id", + TaskType.COMPLETION, + UnifiedCompletionRequest.of(null), + TimeValue.timeValueSeconds(10) + ); + var exception = request.validate(); + assertThat(exception.getMessage(), is("Validation Failed: 1: Field [messages] cannot be null;")); + } + + public void testValidation_ReturnsException_When_UnifiedCompletionRequest_Is_EmptyArray() { + var request = new UnifiedCompletionAction.Request( + "inference_id", + TaskType.COMPLETION, + UnifiedCompletionRequest.of(List.of()), + TimeValue.timeValueSeconds(10) + ); + var exception = request.validate(); + assertThat(exception.getMessage(), is("Validation Failed: 1: Field [messages] cannot be an empty array;")); + } + + public void testValidation_ReturnsException_When_TaskType_IsNot_Completion() { + var request = new UnifiedCompletionAction.Request( + "inference_id", + TaskType.SPARSE_EMBEDDING, + UnifiedCompletionRequest.of(List.of(UnifiedCompletionRequestTests.randomMessage())), + TimeValue.timeValueSeconds(10) + ); + var exception = request.validate(); + assertThat(exception.getMessage(), is("Validation Failed: 1: Field [taskType] must be [completion];")); + } + + public void testValidation_ReturnsNull_When_TaskType_IsAny() { + var request = new UnifiedCompletionAction.Request( + "inference_id", + TaskType.ANY, + UnifiedCompletionRequest.of(List.of(UnifiedCompletionRequestTests.randomMessage())), + TimeValue.timeValueSeconds(10) + ); + assertNull(request.validate()); + } + + @Override + protected UnifiedCompletionAction.Request mutateInstanceForVersion(UnifiedCompletionAction.Request instance, TransportVersion version) { + return instance; + } + + @Override + protected Writeable.Reader instanceReader() { + return UnifiedCompletionAction.Request::new; + } + + @Override + protected UnifiedCompletionAction.Request createTestInstance() { + return new UnifiedCompletionAction.Request( + randomAlphaOfLength(10), + randomFrom(TaskType.values()), + UnifiedCompletionRequestTests.randomUnifiedCompletionRequest(), + TimeValue.timeValueMillis(randomLongBetween(1, 2048)) + ); + } + + @Override + protected UnifiedCompletionAction.Request mutateInstance(UnifiedCompletionAction.Request instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(UnifiedCompletionRequest.getNamedWriteables()); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionRequestTests.java new file mode 100644 index 0000000000000..47a0814a584b7 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/UnifiedCompletionRequestTests.java @@ -0,0 +1,293 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.inference.UnifiedCompletionRequest; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.is; + +public class UnifiedCompletionRequestTests extends AbstractBWCWireSerializationTestCase { + + public void testParseAllFields() throws IOException { + String requestJson = """ + { + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": [ + { + "text": "some text", + "type": "string" + } + ], + "name": "a name", + "tool_call_id": "100", + "tool_calls": [ + { + "id": "call_62136354", + "type": "function", + "function": { + "arguments": "{'order_id': 'order_12345'}", + "name": "get_delivery_date" + } + } + ] + } + ], + "max_completion_tokens": 100, + "stop": ["stop"], + "temperature": 0.1, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object" + } + } + } + ], + "tool_choice": { + "type": "function", + "function": { + "name": "some function" + } + }, + "top_p": 0.2 + } + """; + + try (var parser = createParser(JsonXContent.jsonXContent, requestJson)) { + var request = UnifiedCompletionRequest.PARSER.apply(parser, null); + var expected = new UnifiedCompletionRequest( + List.of( + new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentObjects( + List.of(new UnifiedCompletionRequest.ContentObject("some text", "string")) + ), + "user", + "a name", + "100", + List.of( + new UnifiedCompletionRequest.ToolCall( + "call_62136354", + new UnifiedCompletionRequest.ToolCall.FunctionField("{'order_id': 'order_12345'}", "get_delivery_date"), + "function" + ) + ) + ) + ), + "gpt-4o", + 100L, + List.of("stop"), + 0.1F, + new UnifiedCompletionRequest.ToolChoiceObject( + "function", + new UnifiedCompletionRequest.ToolChoiceObject.FunctionField("some function") + ), + List.of( + new UnifiedCompletionRequest.Tool( + "function", + new UnifiedCompletionRequest.Tool.FunctionField( + "Get the current weather in a given location", + "get_current_weather", + Map.of("type", "object"), + null + ) + ) + ), + 0.2F + ); + + assertThat(request, is(expected)); + } + } + + public void testParsing() throws IOException { + String requestJson = """ + { + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": "What is the weather like in Boston today?" + } + ], + "stop": "none", + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object" + } + } + } + ], + "tool_choice": "auto" + } + """; + + try (var parser = createParser(JsonXContent.jsonXContent, requestJson)) { + var request = UnifiedCompletionRequest.PARSER.apply(parser, null); + var expected = new UnifiedCompletionRequest( + List.of( + new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString("What is the weather like in Boston today?"), + "user", + null, + null, + null + ) + ), + "gpt-4o", + null, + List.of("none"), + null, + new UnifiedCompletionRequest.ToolChoiceString("auto"), + List.of( + new UnifiedCompletionRequest.Tool( + "function", + new UnifiedCompletionRequest.Tool.FunctionField( + "Get the current weather in a given location", + "get_current_weather", + Map.of("type", "object"), + null + ) + ) + ), + null + ); + + assertThat(request, is(expected)); + } + } + + public static UnifiedCompletionRequest randomUnifiedCompletionRequest() { + return new UnifiedCompletionRequest( + randomList(5, UnifiedCompletionRequestTests::randomMessage), + randomAlphaOfLengthOrNull(10), + randomPositiveLongOrNull(), + randomStopOrNull(), + randomFloatOrNull(), + randomToolChoiceOrNull(), + randomToolListOrNull(), + randomFloatOrNull() + ); + } + + public static UnifiedCompletionRequest.Message randomMessage() { + return new UnifiedCompletionRequest.Message( + randomContent(), + randomAlphaOfLength(10), + randomAlphaOfLengthOrNull(10), + randomAlphaOfLengthOrNull(10), + randomToolCallListOrNull() + ); + } + + public static UnifiedCompletionRequest.Content randomContent() { + return randomBoolean() + ? new UnifiedCompletionRequest.ContentString(randomAlphaOfLength(10)) + : new UnifiedCompletionRequest.ContentObjects(randomList(10, UnifiedCompletionRequestTests::randomContentObject)); + } + + public static UnifiedCompletionRequest.ContentObject randomContentObject() { + return new UnifiedCompletionRequest.ContentObject(randomAlphaOfLength(10), randomAlphaOfLength(10)); + } + + public static List randomToolCallListOrNull() { + return randomBoolean() ? randomList(10, UnifiedCompletionRequestTests::randomToolCall) : null; + } + + public static UnifiedCompletionRequest.ToolCall randomToolCall() { + return new UnifiedCompletionRequest.ToolCall(randomAlphaOfLength(10), randomToolCallFunctionField(), randomAlphaOfLength(10)); + } + + public static UnifiedCompletionRequest.ToolCall.FunctionField randomToolCallFunctionField() { + return new UnifiedCompletionRequest.ToolCall.FunctionField(randomAlphaOfLength(10), randomAlphaOfLength(10)); + } + + public static List randomStopOrNull() { + return randomBoolean() ? randomStop() : null; + } + + public static List randomStop() { + return randomList(5, () -> randomAlphaOfLength(10)); + } + + public static UnifiedCompletionRequest.ToolChoice randomToolChoiceOrNull() { + return randomBoolean() ? randomToolChoice() : null; + } + + public static UnifiedCompletionRequest.ToolChoice randomToolChoice() { + return randomBoolean() + ? new UnifiedCompletionRequest.ToolChoiceString(randomAlphaOfLength(10)) + : new UnifiedCompletionRequest.ToolChoiceObject(randomAlphaOfLength(10), randomToolChoiceObjectFunctionField()); + } + + public static UnifiedCompletionRequest.ToolChoiceObject.FunctionField randomToolChoiceObjectFunctionField() { + return new UnifiedCompletionRequest.ToolChoiceObject.FunctionField(randomAlphaOfLength(10)); + } + + public static List randomToolListOrNull() { + return randomBoolean() ? randomList(10, UnifiedCompletionRequestTests::randomTool) : null; + } + + public static UnifiedCompletionRequest.Tool randomTool() { + return new UnifiedCompletionRequest.Tool(randomAlphaOfLength(10), randomToolFunctionField()); + } + + public static UnifiedCompletionRequest.Tool.FunctionField randomToolFunctionField() { + return new UnifiedCompletionRequest.Tool.FunctionField( + randomAlphaOfLengthOrNull(10), + randomAlphaOfLength(10), + null, + randomOptionalBoolean() + ); + } + + @Override + protected UnifiedCompletionRequest mutateInstanceForVersion(UnifiedCompletionRequest instance, TransportVersion version) { + return instance; + } + + @Override + protected Writeable.Reader instanceReader() { + return UnifiedCompletionRequest::new; + } + + @Override + protected UnifiedCompletionRequest createTestInstance() { + return randomUnifiedCompletionRequest(); + } + + @Override + protected UnifiedCompletionRequest mutateInstance(UnifiedCompletionRequest instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(UnifiedCompletionRequest.getNamedWriteables()); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResultsTests.java new file mode 100644 index 0000000000000..a8f569dbef9d1 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResultsTests.java @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + * + * this file was contributed to by a generative AI + */ + +package org.elasticsearch.xpack.core.inference.results; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; + +public class StreamingUnifiedChatCompletionResultsTests extends ESTestCase { + + public void testResults_toXContentChunked() throws IOException { + String expected = """ + { + "id": "chunk1", + "choices": [ + { + "delta": { + "content": "example_content", + "refusal": "example_refusal", + "role": "assistant", + "tool_calls": [ + { + "index": 1, + "id": "tool1", + "function": { + "arguments": "example_arguments", + "name": "example_function" + }, + "type": "function" + } + ] + }, + "finish_reason": "example_reason", + "index": 0 + } + ], + "model": "example_model", + "object": "example_object", + "usage": { + "completion_tokens": 10, + "prompt_tokens": 5, + "total_tokens": 15 + } + } + """; + + StreamingUnifiedChatCompletionResults.ChatCompletionChunk chunk = new StreamingUnifiedChatCompletionResults.ChatCompletionChunk( + "chunk1", + List.of( + new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice( + new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta( + "example_content", + "example_refusal", + "assistant", + List.of( + new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall( + 1, + "tool1", + new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall.Function( + "example_arguments", + "example_function" + ), + "function" + ) + ) + ), + "example_reason", + 0 + ) + ), + "example_model", + "example_object", + new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Usage(10, 5, 15) + ); + + Deque deque = new ArrayDeque<>(); + deque.add(chunk); + StreamingUnifiedChatCompletionResults.Results results = new StreamingUnifiedChatCompletionResults.Results(deque); + XContentBuilder builder = JsonXContent.contentBuilder(); + results.toXContentChunked(null).forEachRemaining(xContent -> { + try { + xContent.toXContent(builder, null); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + assertEquals(expected.replaceAll("\\s+", ""), Strings.toString(builder.prettyPrint()).trim()); + } + + public void testChoiceToXContentChunked() throws IOException { + String expected = """ + { + "delta": { + "content": "example_content", + "refusal": "example_refusal", + "role": "assistant", + "tool_calls": [ + { + "index": 1, + "id": "tool1", + "function": { + "arguments": "example_arguments", + "name": "example_function" + }, + "type": "function" + } + ] + }, + "finish_reason": "example_reason", + "index": 0 + } + """; + + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice choice = + new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice( + new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta( + "example_content", + "example_refusal", + "assistant", + List.of( + new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall( + 1, + "tool1", + new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall.Function( + "example_arguments", + "example_function" + ), + "function" + ) + ) + ), + "example_reason", + 0 + ); + + XContentBuilder builder = JsonXContent.contentBuilder(); + choice.toXContentChunked(null).forEachRemaining(xContent -> { + try { + xContent.toXContent(builder, null); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + assertEquals(expected.replaceAll("\\s+", ""), Strings.toString(builder.prettyPrint()).trim()); + } + + public void testToolCallToXContentChunked() throws IOException { + String expected = """ + { + "index": 1, + "id": "tool1", + "function": { + "arguments": "example_arguments", + "name": "example_function" + }, + "type": "function" + } + """; + + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall toolCall = + new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall( + 1, + "tool1", + new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall.Function( + "example_arguments", + "example_function" + ), + "function" + ); + + XContentBuilder builder = JsonXContent.contentBuilder(); + toolCall.toXContentChunked(null).forEachRemaining(xContent -> { + try { + xContent.toXContent(builder, null); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + assertEquals(expected.replaceAll("\\s+", ""), Strings.toString(builder.prettyPrint()).trim()); + } + +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 17579fd6368ce..eeffa1db54856 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -4175,6 +4175,7 @@ public void testInferenceUserRole() { assertTrue(role.cluster().check("cluster:monitor/xpack/inference", request, authentication)); assertTrue(role.cluster().check("cluster:monitor/xpack/inference/get", request, authentication)); assertFalse(role.cluster().check("cluster:admin/xpack/inference/put", request, authentication)); + assertTrue(role.cluster().check("cluster:monitor/xpack/inference/unified", request, authentication)); assertFalse(role.cluster().check("cluster:admin/xpack/inference/delete", request, authentication)); assertTrue(role.cluster().check("cluster:monitor/xpack/ml/trained_models/deployment/infer", request, authentication)); assertFalse(role.cluster().check("cluster:admin/xpack/ml/trained_models/deployment/start", request, authentication)); diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java index 86c0128a3e53c..1716057cdfe46 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java @@ -21,6 +21,9 @@ import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent; import org.junit.ClassRule; @@ -341,10 +344,21 @@ protected Deque streamInferOnMockService(String modelId, TaskTy return callAsync(endpoint, input); } + protected Deque unifiedCompletionInferOnMockService(String modelId, TaskType taskType, List input) + throws Exception { + var endpoint = Strings.format("_inference/%s/%s/_unified", taskType, modelId); + return callAsyncUnified(endpoint, input, "user"); + } + private Deque callAsync(String endpoint, List input) throws Exception { - var responseConsumer = new AsyncInferenceResponseConsumer(); var request = new Request("POST", endpoint); request.setJsonEntity(jsonBody(input, null)); + + return execAsyncCall(request); + } + + private Deque execAsyncCall(Request request) throws Exception { + var responseConsumer = new AsyncInferenceResponseConsumer(); request.setOptions(RequestOptions.DEFAULT.toBuilder().setHttpAsyncResponseConsumerFactory(() -> responseConsumer).build()); var latch = new CountDownLatch(1); client().performRequestAsync(request, new ResponseListener() { @@ -362,6 +376,22 @@ public void onFailure(Exception exception) { return responseConsumer.events(); } + private Deque callAsyncUnified(String endpoint, List input, String role) throws Exception { + var request = new Request("POST", endpoint); + + request.setJsonEntity(createUnifiedJsonBody(input, role)); + return execAsyncCall(request); + } + + private String createUnifiedJsonBody(List input, String role) throws IOException { + var messages = input.stream().map(i -> Map.of("content", i, "role", role)).toList(); + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + builder.startObject(); + builder.field("messages", messages); + builder.endObject(); + return org.elasticsearch.common.Strings.toString(builder); + } + protected Map infer(String modelId, TaskType taskType, List input) throws IOException { var endpoint = Strings.format("_inference/%s/%s", taskType, modelId); return inferInternal(endpoint, input, null, Map.of()); diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index 604e1d4f553b2..2099ec8287a76 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -11,13 +11,18 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceFeature; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -481,6 +486,56 @@ public void testSupportedStream() throws Exception { } } + public void testUnifiedCompletionInference() throws Exception { + String modelId = "streaming"; + putModel(modelId, mockCompletionServiceModelConfig(TaskType.COMPLETION)); + var singleModel = getModel(modelId); + assertEquals(modelId, singleModel.get("inference_id")); + assertEquals(TaskType.COMPLETION.toString(), singleModel.get("task_type")); + + var input = IntStream.range(1, 2 + randomInt(8)).mapToObj(i -> randomUUID()).toList(); + try { + var events = unifiedCompletionInferOnMockService(modelId, TaskType.COMPLETION, input); + var expectedResponses = expectedResultsIterator(input); + assertThat(events.size(), equalTo((input.size() + 1) * 2)); + events.forEach(event -> { + switch (event.name()) { + case EVENT -> assertThat(event.value(), equalToIgnoringCase("message")); + case DATA -> assertThat(event.value(), equalTo(expectedResponses.next())); + } + }); + } finally { + deleteModel(modelId); + } + } + + private static Iterator expectedResultsIterator(List input) { + return Stream.concat(input.stream().map(String::toUpperCase).map(InferenceCrudIT::expectedResult), Stream.of("[DONE]")).iterator(); + } + + private static String expectedResult(String input) { + try { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + builder.startObject(); + builder.field("id", "id"); + builder.startArray("choices"); + builder.startObject(); + builder.startObject("delta"); + builder.field("content", input); + builder.endObject(); + builder.field("index", 0); + builder.endObject(); + builder.endArray(); + builder.field("model", "gpt-4o-2024-08-06"); + builder.field("object", "chat.completion.chunk"); + builder.endObject(); + + return Strings.toString(builder); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public void testGetZeroModels() throws IOException { var models = getModels("_all", TaskType.COMPLETION); assertThat(models, empty()); diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java index ae11a02d312e2..f5f682b143a72 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java @@ -31,6 +31,7 @@ import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnifiedCompletionRequest; import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; @@ -132,6 +133,16 @@ public void infer( } } + @Override + public void unifiedCompletionInfer( + Model model, + UnifiedCompletionRequest request, + TimeValue timeout, + ActionListener listener + ) { + listener.onFailure(new UnsupportedOperationException("unifiedCompletionInfer not supported")); + } + @Override public void chunkedInfer( Model model, diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java index 9320571572f0a..fa1e27005c287 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java @@ -29,6 +29,7 @@ import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnifiedCompletionRequest; import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; @@ -120,6 +121,16 @@ public void infer( } } + @Override + public void unifiedCompletionInfer( + Model model, + UnifiedCompletionRequest request, + TimeValue timeout, + ActionListener listener + ) { + listener.onFailure(new UnsupportedOperationException("unifiedCompletionInfer not supported")); + } + @Override public void chunkedInfer( Model model, diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java index fe0223cce0323..64569fd8c5c6a 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java @@ -29,6 +29,7 @@ import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnifiedCompletionRequest; import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; @@ -123,6 +124,16 @@ public void infer( } } + @Override + public void unifiedCompletionInfer( + Model model, + UnifiedCompletionRequest request, + TimeValue timeout, + ActionListener listener + ) { + throw new UnsupportedOperationException("unifiedCompletionInfer not supported"); + } + @Override public void chunkedInfer( Model model, diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java index 6d7983bc8cb53..f7a05a27354ef 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java @@ -30,12 +30,14 @@ import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnifiedCompletionRequest; import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; +import org.elasticsearch.xpack.core.inference.results.StreamingUnifiedChatCompletionResults; import java.io.IOException; import java.util.EnumSet; @@ -121,6 +123,24 @@ public void infer( } } + @Override + public void unifiedCompletionInfer( + Model model, + UnifiedCompletionRequest request, + TimeValue timeout, + ActionListener listener + ) { + switch (model.getConfigurations().getTaskType()) { + case COMPLETION -> listener.onResponse(makeUnifiedResults(request)); + default -> listener.onFailure( + new ElasticsearchStatusException( + TaskType.unsupportedTaskTypeErrorMsg(model.getConfigurations().getTaskType(), name()), + RestStatus.BAD_REQUEST + ) + ); + } + } + private StreamingChatCompletionResults makeResults(List input) { var responseIter = input.stream().map(String::toUpperCase).iterator(); return new StreamingChatCompletionResults(subscriber -> { @@ -152,6 +172,59 @@ private ChunkedToXContent completionChunk(String delta) { ); } + private StreamingUnifiedChatCompletionResults makeUnifiedResults(UnifiedCompletionRequest request) { + var responseIter = request.messages().stream().map(message -> message.content().toString().toUpperCase()).iterator(); + return new StreamingUnifiedChatCompletionResults(subscriber -> { + subscriber.onSubscribe(new Flow.Subscription() { + @Override + public void request(long n) { + if (responseIter.hasNext()) { + subscriber.onNext(unifiedCompletionChunk(responseIter.next())); + } else { + subscriber.onComplete(); + } + } + + @Override + public void cancel() {} + }); + }); + } + + /* + The response format looks like this + { + "id": "chatcmpl-AarrzyuRflye7yzDF4lmVnenGmQCF", + "choices": [ + { + "delta": { + "content": " information" + }, + "index": 0 + } + ], + "model": "gpt-4o-2024-08-06", + "object": "chat.completion.chunk" + } + */ + private ChunkedToXContent unifiedCompletionChunk(String delta) { + return params -> Iterators.concat( + ChunkedToXContentHelper.startObject(), + ChunkedToXContentHelper.field("id", "id"), + ChunkedToXContentHelper.startArray("choices"), + ChunkedToXContentHelper.startObject(), + ChunkedToXContentHelper.startObject("delta"), + ChunkedToXContentHelper.field("content", delta), + ChunkedToXContentHelper.endObject(), + ChunkedToXContentHelper.field("index", 0), + ChunkedToXContentHelper.endObject(), + ChunkedToXContentHelper.endArray(), + ChunkedToXContentHelper.field("model", "gpt-4o-2024-08-06"), + ChunkedToXContentHelper.field("object", "chat.completion.chunk"), + ChunkedToXContentHelper.endObject() + ); + } + @Override public void chunkedInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java index 673b841317a3d..a4187f4c4fa90 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java @@ -16,6 +16,7 @@ import org.elasticsearch.inference.SecretSettings; import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.inference.UnifiedCompletionRequest; import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; @@ -137,11 +138,18 @@ public static List getNamedWriteables() { addEisNamedWriteables(namedWriteables); addAlibabaCloudSearchNamedWriteables(namedWriteables); + addUnifiedNamedWriteables(namedWriteables); + namedWriteables.addAll(StreamingTaskManager.namedWriteables()); return namedWriteables; } + private static void addUnifiedNamedWriteables(List namedWriteables) { + var writeables = UnifiedCompletionRequest.getNamedWriteables(); + namedWriteables.addAll(writeables); + } + private static void addAmazonBedrockNamedWriteables(List namedWriteables) { namedWriteables.add( new NamedWriteableRegistry.Entry( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index d7d623ab20143..148a784456361 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -51,6 +51,7 @@ import org.elasticsearch.xpack.core.inference.action.GetInferenceServicesAction; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.action.PutInferenceModelAction; +import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction; import org.elasticsearch.xpack.inference.action.TransportDeleteInferenceEndpointAction; import org.elasticsearch.xpack.inference.action.TransportGetInferenceDiagnosticsAction; @@ -59,6 +60,7 @@ import org.elasticsearch.xpack.inference.action.TransportInferenceAction; import org.elasticsearch.xpack.inference.action.TransportInferenceUsageAction; import org.elasticsearch.xpack.inference.action.TransportPutInferenceModelAction; +import org.elasticsearch.xpack.inference.action.TransportUnifiedCompletionInferenceAction; import org.elasticsearch.xpack.inference.action.TransportUpdateInferenceModelAction; import org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilter; import org.elasticsearch.xpack.inference.common.Truncator; @@ -86,6 +88,7 @@ import org.elasticsearch.xpack.inference.rest.RestInferenceAction; import org.elasticsearch.xpack.inference.rest.RestPutInferenceModelAction; import org.elasticsearch.xpack.inference.rest.RestStreamInferenceAction; +import org.elasticsearch.xpack.inference.rest.RestUnifiedCompletionInferenceAction; import org.elasticsearch.xpack.inference.rest.RestUpdateInferenceModelAction; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchService; @@ -159,8 +162,9 @@ public InferencePlugin(Settings settings) { @Override public List> getActions() { - return List.of( + var availableActions = List.of( new ActionHandler<>(InferenceAction.INSTANCE, TransportInferenceAction.class), + new ActionHandler<>(GetInferenceModelAction.INSTANCE, TransportGetInferenceModelAction.class), new ActionHandler<>(PutInferenceModelAction.INSTANCE, TransportPutInferenceModelAction.class), new ActionHandler<>(UpdateInferenceModelAction.INSTANCE, TransportUpdateInferenceModelAction.class), @@ -169,6 +173,13 @@ public InferencePlugin(Settings settings) { new ActionHandler<>(GetInferenceDiagnosticsAction.INSTANCE, TransportGetInferenceDiagnosticsAction.class), new ActionHandler<>(GetInferenceServicesAction.INSTANCE, TransportGetInferenceServicesAction.class) ); + + List> conditionalActions = + UnifiedCompletionFeature.UNIFIED_COMPLETION_FEATURE_FLAG.isEnabled() + ? List.of(new ActionHandler<>(UnifiedCompletionAction.INSTANCE, TransportUnifiedCompletionInferenceAction.class)) + : List.of(); + + return Stream.concat(availableActions.stream(), conditionalActions.stream()).toList(); } @Override @@ -183,7 +194,7 @@ public List getRestHandlers( Supplier nodesInCluster, Predicate clusterSupportsFeature ) { - return List.of( + var availableRestActions = List.of( new RestInferenceAction(), new RestStreamInferenceAction(), new RestGetInferenceModelAction(), @@ -193,6 +204,11 @@ public List getRestHandlers( new RestGetInferenceDiagnosticsAction(), new RestGetInferenceServicesAction() ); + List conditionalRestActions = UnifiedCompletionFeature.UNIFIED_COMPLETION_FEATURE_FLAG.isEnabled() + ? List.of(new RestUnifiedCompletionInferenceAction()) + : List.of(); + + return Stream.concat(availableRestActions.stream(), conditionalRestActions.stream()).toList(); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/UnifiedCompletionFeature.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/UnifiedCompletionFeature.java new file mode 100644 index 0000000000000..3e13d0c1e39de --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/UnifiedCompletionFeature.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference; + +import org.elasticsearch.common.util.FeatureFlag; + +/** + * Unified Completion feature flag. When the feature is complete, this flag will be removed. + * Enable feature via JVM option: `-Des.inference_unified_feature_flag_enabled=true`. + */ +public class UnifiedCompletionFeature { + public static final FeatureFlag UNIFIED_COMPLETION_FEATURE_FLAG = new FeatureFlag("inference_unified"); + + private UnifiedCompletionFeature() {} +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java new file mode 100644 index 0000000000000..2a0e8e1775279 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java @@ -0,0 +1,250 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InferenceService; +import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.inference.action.BaseInferenceActionRequest; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; +import org.elasticsearch.xpack.inference.common.DelegatingProcessor; +import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.elasticsearch.xpack.inference.telemetry.InferenceStats; +import org.elasticsearch.xpack.inference.telemetry.InferenceTimer; + +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.xpack.inference.telemetry.InferenceStats.modelAttributes; +import static org.elasticsearch.xpack.inference.telemetry.InferenceStats.responseAttributes; + +public abstract class BaseTransportInferenceAction extends HandledTransportAction< + Request, + InferenceAction.Response> { + + private static final Logger log = LogManager.getLogger(BaseTransportInferenceAction.class); + private static final String STREAMING_INFERENCE_TASK_TYPE = "streaming_inference"; + private static final String STREAMING_TASK_ACTION = "xpack/inference/streaming_inference[n]"; + private final ModelRegistry modelRegistry; + private final InferenceServiceRegistry serviceRegistry; + private final InferenceStats inferenceStats; + private final StreamingTaskManager streamingTaskManager; + + public BaseTransportInferenceAction( + String inferenceActionName, + TransportService transportService, + ActionFilters actionFilters, + ModelRegistry modelRegistry, + InferenceServiceRegistry serviceRegistry, + InferenceStats inferenceStats, + StreamingTaskManager streamingTaskManager, + Writeable.Reader requestReader + ) { + super(inferenceActionName, transportService, actionFilters, requestReader, EsExecutors.DIRECT_EXECUTOR_SERVICE); + this.modelRegistry = modelRegistry; + this.serviceRegistry = serviceRegistry; + this.inferenceStats = inferenceStats; + this.streamingTaskManager = streamingTaskManager; + } + + @Override + protected void doExecute(Task task, Request request, ActionListener listener) { + var timer = InferenceTimer.start(); + + var getModelListener = ActionListener.wrap((UnparsedModel unparsedModel) -> { + var service = serviceRegistry.getService(unparsedModel.service()); + try { + validationHelper(service::isEmpty, () -> unknownServiceException(unparsedModel.service(), request.getInferenceEntityId())); + validationHelper( + () -> request.getTaskType().isAnyOrSame(unparsedModel.taskType()) == false, + () -> requestModelTaskTypeMismatchException(request.getTaskType(), unparsedModel.taskType()) + ); + validationHelper( + () -> isInvalidTaskTypeForInferenceEndpoint(request, unparsedModel), + () -> createInvalidTaskTypeException(request, unparsedModel) + ); + } catch (Exception e) { + recordMetrics(unparsedModel, timer, e); + listener.onFailure(e); + return; + } + + var model = service.get() + .parsePersistedConfigWithSecrets( + unparsedModel.inferenceEntityId(), + unparsedModel.taskType(), + unparsedModel.settings(), + unparsedModel.secrets() + ); + inferOnServiceWithMetrics(model, request, service.get(), timer, listener); + }, e -> { + try { + inferenceStats.inferenceDuration().record(timer.elapsedMillis(), responseAttributes(e)); + } catch (Exception metricsException) { + log.atDebug().withThrowable(metricsException).log("Failed to record metrics when the model is missing, dropping metrics"); + } + listener.onFailure(e); + }); + + modelRegistry.getModelWithSecrets(request.getInferenceEntityId(), getModelListener); + } + + private static void validationHelper(Supplier validationFailure, Supplier exceptionCreator) { + if (validationFailure.get()) { + throw exceptionCreator.get(); + } + } + + protected abstract boolean isInvalidTaskTypeForInferenceEndpoint(Request request, UnparsedModel unparsedModel); + + protected abstract ElasticsearchStatusException createInvalidTaskTypeException(Request request, UnparsedModel unparsedModel); + + private void recordMetrics(UnparsedModel model, InferenceTimer timer, @Nullable Throwable t) { + try { + inferenceStats.inferenceDuration().record(timer.elapsedMillis(), responseAttributes(model, t)); + } catch (Exception e) { + log.atDebug().withThrowable(e).log("Failed to record metrics with an unparsed model, dropping metrics"); + } + } + + private void inferOnServiceWithMetrics( + Model model, + Request request, + InferenceService service, + InferenceTimer timer, + ActionListener listener + ) { + inferenceStats.requestCount().incrementBy(1, modelAttributes(model)); + inferOnService(model, request, service, ActionListener.wrap(inferenceResults -> { + if (request.isStreaming()) { + var taskProcessor = streamingTaskManager.create(STREAMING_INFERENCE_TASK_TYPE, STREAMING_TASK_ACTION); + inferenceResults.publisher().subscribe(taskProcessor); + + var instrumentedStream = new PublisherWithMetrics(timer, model); + taskProcessor.subscribe(instrumentedStream); + + listener.onResponse(new InferenceAction.Response(inferenceResults, instrumentedStream)); + } else { + recordMetrics(model, timer, null); + listener.onResponse(new InferenceAction.Response(inferenceResults)); + } + }, e -> { + recordMetrics(model, timer, e); + listener.onFailure(e); + })); + } + + private void recordMetrics(Model model, InferenceTimer timer, @Nullable Throwable t) { + try { + inferenceStats.inferenceDuration().record(timer.elapsedMillis(), responseAttributes(model, t)); + } catch (Exception e) { + log.atDebug().withThrowable(e).log("Failed to record metrics with a parsed model, dropping metrics"); + } + } + + private void inferOnService(Model model, Request request, InferenceService service, ActionListener listener) { + if (request.isStreaming() == false || service.canStream(request.getTaskType())) { + doInference(model, request, service, listener); + } else { + listener.onFailure(unsupportedStreamingTaskException(request, service)); + } + } + + protected abstract void doInference( + Model model, + Request request, + InferenceService service, + ActionListener listener + ); + + private ElasticsearchStatusException unsupportedStreamingTaskException(Request request, InferenceService service) { + var supportedTasks = service.supportedStreamingTasks(); + if (supportedTasks.isEmpty()) { + return new ElasticsearchStatusException( + format("Streaming is not allowed for service [%s].", service.name()), + RestStatus.METHOD_NOT_ALLOWED + ); + } else { + var validTasks = supportedTasks.stream().map(TaskType::toString).collect(Collectors.joining(",")); + return new ElasticsearchStatusException( + format( + "Streaming is not allowed for service [%s] and task [%s]. Supported tasks: [%s]", + service.name(), + request.getTaskType(), + validTasks + ), + RestStatus.METHOD_NOT_ALLOWED + ); + } + } + + private static ElasticsearchStatusException unknownServiceException(String service, String inferenceId) { + return new ElasticsearchStatusException("Unknown service [{}] for model [{}]. ", RestStatus.BAD_REQUEST, service, inferenceId); + } + + private static ElasticsearchStatusException requestModelTaskTypeMismatchException(TaskType requested, TaskType expected) { + return new ElasticsearchStatusException( + "Incompatible task_type, the requested type [{}] does not match the model type [{}]", + RestStatus.BAD_REQUEST, + requested, + expected + ); + } + + private class PublisherWithMetrics extends DelegatingProcessor { + + private final InferenceTimer timer; + private final Model model; + + private PublisherWithMetrics(InferenceTimer timer, Model model) { + this.timer = timer; + this.model = model; + } + + @Override + protected void next(ChunkedToXContent item) { + downstream().onNext(item); + } + + @Override + public void onError(Throwable throwable) { + recordMetrics(model, timer, throwable); + super.onError(throwable); + } + + @Override + protected void onCancel() { + recordMetrics(model, timer, null); + super.onCancel(); + } + + @Override + public void onComplete() { + recordMetrics(model, timer, null); + super.onComplete(); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java index ba9ab3c133731..08e6d869a553d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java @@ -7,47 +7,22 @@ package org.elasticsearch.xpack.inference.action; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.xcontent.ChunkedToXContent; -import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.InferenceServiceRegistry; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.Model; -import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.injection.guice.Inject; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; -import org.elasticsearch.xpack.inference.common.DelegatingProcessor; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.telemetry.InferenceStats; -import org.elasticsearch.xpack.inference.telemetry.InferenceTimer; -import java.util.stream.Collectors; - -import static org.elasticsearch.core.Strings.format; -import static org.elasticsearch.xpack.inference.telemetry.InferenceStats.modelAttributes; -import static org.elasticsearch.xpack.inference.telemetry.InferenceStats.responseAttributes; - -public class TransportInferenceAction extends HandledTransportAction { - private static final Logger log = LogManager.getLogger(TransportInferenceAction.class); - private static final String STREAMING_INFERENCE_TASK_TYPE = "streaming_inference"; - private static final String STREAMING_TASK_ACTION = "xpack/inference/streaming_inference[n]"; - - private final ModelRegistry modelRegistry; - private final InferenceServiceRegistry serviceRegistry; - private final InferenceStats inferenceStats; - private final StreamingTaskManager streamingTaskManager; +public class TransportInferenceAction extends BaseTransportInferenceAction { @Inject public TransportInferenceAction( @@ -58,184 +33,44 @@ public TransportInferenceAction( InferenceStats inferenceStats, StreamingTaskManager streamingTaskManager ) { - super(InferenceAction.NAME, transportService, actionFilters, InferenceAction.Request::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); - this.modelRegistry = modelRegistry; - this.serviceRegistry = serviceRegistry; - this.inferenceStats = inferenceStats; - this.streamingTaskManager = streamingTaskManager; + super( + InferenceAction.NAME, + transportService, + actionFilters, + modelRegistry, + serviceRegistry, + inferenceStats, + streamingTaskManager, + InferenceAction.Request::new + ); } @Override - protected void doExecute(Task task, InferenceAction.Request request, ActionListener listener) { - var timer = InferenceTimer.start(); - - var getModelListener = ActionListener.wrap((UnparsedModel unparsedModel) -> { - var service = serviceRegistry.getService(unparsedModel.service()); - if (service.isEmpty()) { - var e = unknownServiceException(unparsedModel.service(), request.getInferenceEntityId()); - recordMetrics(unparsedModel, timer, e); - listener.onFailure(e); - return; - } - - if (request.getTaskType().isAnyOrSame(unparsedModel.taskType()) == false) { - // not the wildcard task type and not the model task type - var e = incompatibleTaskTypeException(request.getTaskType(), unparsedModel.taskType()); - recordMetrics(unparsedModel, timer, e); - listener.onFailure(e); - return; - } - - var model = service.get() - .parsePersistedConfigWithSecrets( - unparsedModel.inferenceEntityId(), - unparsedModel.taskType(), - unparsedModel.settings(), - unparsedModel.secrets() - ); - inferOnServiceWithMetrics(model, request, service.get(), timer, listener); - }, e -> { - try { - inferenceStats.inferenceDuration().record(timer.elapsedMillis(), responseAttributes(e)); - } catch (Exception metricsException) { - log.atDebug().withThrowable(metricsException).log("Failed to record metrics when the model is missing, dropping metrics"); - } - listener.onFailure(e); - }); - - modelRegistry.getModelWithSecrets(request.getInferenceEntityId(), getModelListener); - } - - private void recordMetrics(UnparsedModel model, InferenceTimer timer, @Nullable Throwable t) { - try { - inferenceStats.inferenceDuration().record(timer.elapsedMillis(), responseAttributes(model, t)); - } catch (Exception e) { - log.atDebug().withThrowable(e).log("Failed to record metrics with an unparsed model, dropping metrics"); - } - } - - private void inferOnServiceWithMetrics( - Model model, - InferenceAction.Request request, - InferenceService service, - InferenceTimer timer, - ActionListener listener - ) { - inferenceStats.requestCount().incrementBy(1, modelAttributes(model)); - inferOnService(model, request, service, ActionListener.wrap(inferenceResults -> { - if (request.isStreaming()) { - var taskProcessor = streamingTaskManager.create(STREAMING_INFERENCE_TASK_TYPE, STREAMING_TASK_ACTION); - inferenceResults.publisher().subscribe(taskProcessor); - - var instrumentedStream = new PublisherWithMetrics(timer, model); - taskProcessor.subscribe(instrumentedStream); - - listener.onResponse(new InferenceAction.Response(inferenceResults, instrumentedStream)); - } else { - recordMetrics(model, timer, null); - listener.onResponse(new InferenceAction.Response(inferenceResults)); - } - }, e -> { - recordMetrics(model, timer, e); - listener.onFailure(e); - })); + protected boolean isInvalidTaskTypeForInferenceEndpoint(InferenceAction.Request request, UnparsedModel unparsedModel) { + return false; } - private void recordMetrics(Model model, InferenceTimer timer, @Nullable Throwable t) { - try { - inferenceStats.inferenceDuration().record(timer.elapsedMillis(), responseAttributes(model, t)); - } catch (Exception e) { - log.atDebug().withThrowable(e).log("Failed to record metrics with a parsed model, dropping metrics"); - } + @Override + protected ElasticsearchStatusException createInvalidTaskTypeException(InferenceAction.Request request, UnparsedModel unparsedModel) { + return null; } - private void inferOnService( + @Override + protected void doInference( Model model, InferenceAction.Request request, InferenceService service, ActionListener listener ) { - if (request.isStreaming() == false || service.canStream(request.getTaskType())) { - service.infer( - model, - request.getQuery(), - request.getInput(), - request.isStreaming(), - request.getTaskSettings(), - request.getInputType(), - request.getInferenceTimeout(), - listener - ); - } else { - listener.onFailure(unsupportedStreamingTaskException(request, service)); - } - } - - private ElasticsearchStatusException unsupportedStreamingTaskException(InferenceAction.Request request, InferenceService service) { - var supportedTasks = service.supportedStreamingTasks(); - if (supportedTasks.isEmpty()) { - return new ElasticsearchStatusException( - format("Streaming is not allowed for service [%s].", service.name()), - RestStatus.METHOD_NOT_ALLOWED - ); - } else { - var validTasks = supportedTasks.stream().map(TaskType::toString).collect(Collectors.joining(",")); - return new ElasticsearchStatusException( - format( - "Streaming is not allowed for service [%s] and task [%s]. Supported tasks: [%s]", - service.name(), - request.getTaskType(), - validTasks - ), - RestStatus.METHOD_NOT_ALLOWED - ); - } - } - - private static ElasticsearchStatusException unknownServiceException(String service, String inferenceId) { - return new ElasticsearchStatusException("Unknown service [{}] for model [{}]. ", RestStatus.BAD_REQUEST, service, inferenceId); - } - - private static ElasticsearchStatusException incompatibleTaskTypeException(TaskType requested, TaskType expected) { - return new ElasticsearchStatusException( - "Incompatible task_type, the requested type [{}] does not match the model type [{}]", - RestStatus.BAD_REQUEST, - requested, - expected + service.infer( + model, + request.getQuery(), + request.getInput(), + request.isStreaming(), + request.getTaskSettings(), + request.getInputType(), + request.getInferenceTimeout(), + listener ); } - - private class PublisherWithMetrics extends DelegatingProcessor { - private final InferenceTimer timer; - private final Model model; - - private PublisherWithMetrics(InferenceTimer timer, Model model) { - this.timer = timer; - this.model = model; - } - - @Override - protected void next(ChunkedToXContent item) { - downstream().onNext(item); - } - - @Override - public void onError(Throwable throwable) { - recordMetrics(model, timer, throwable); - super.onError(throwable); - } - - @Override - protected void onCancel() { - recordMetrics(model, timer, null); - super.onCancel(); - } - - @Override - public void onComplete() { - recordMetrics(model, timer, null); - super.onComplete(); - } - } - } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java new file mode 100644 index 0000000000000..f0906231d8f42 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.inference.InferenceService; +import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; +import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; +import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.elasticsearch.xpack.inference.telemetry.InferenceStats; + +public class TransportUnifiedCompletionInferenceAction extends BaseTransportInferenceAction { + + @Inject + public TransportUnifiedCompletionInferenceAction( + TransportService transportService, + ActionFilters actionFilters, + ModelRegistry modelRegistry, + InferenceServiceRegistry serviceRegistry, + InferenceStats inferenceStats, + StreamingTaskManager streamingTaskManager + ) { + super( + UnifiedCompletionAction.NAME, + transportService, + actionFilters, + modelRegistry, + serviceRegistry, + inferenceStats, + streamingTaskManager, + UnifiedCompletionAction.Request::new + ); + } + + @Override + protected boolean isInvalidTaskTypeForInferenceEndpoint(UnifiedCompletionAction.Request request, UnparsedModel unparsedModel) { + return request.getTaskType().isAnyOrSame(TaskType.COMPLETION) == false || unparsedModel.taskType() != TaskType.COMPLETION; + } + + @Override + protected ElasticsearchStatusException createInvalidTaskTypeException( + UnifiedCompletionAction.Request request, + UnparsedModel unparsedModel + ) { + return new ElasticsearchStatusException( + "Incompatible task_type for unified API, the requested type [{}] must be one of [{}]", + RestStatus.BAD_REQUEST, + request.getTaskType(), + TaskType.COMPLETION.toString() + ); + } + + @Override + protected void doInference( + Model model, + UnifiedCompletionAction.Request request, + InferenceService service, + ActionListener listener + ) { + service.unifiedCompletionInfer(model, request.getUnifiedCompletionRequest(), null, listener); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java index 03e794e42c3a2..eda3fc0f3bfdb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java @@ -9,7 +9,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventField; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; import java.util.concurrent.Flow; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -25,6 +32,33 @@ public abstract class DelegatingProcessor implements Flow.Processor private Flow.Subscriber downstream; private Flow.Subscription upstream; + public static Deque parseEvent( + Deque item, + ParseChunkFunction parseFunction, + XContentParserConfiguration parserConfig, + Logger logger + ) throws Exception { + var results = new ArrayDeque(item.size()); + for (ServerSentEvent event : item) { + if (ServerSentEventField.DATA == event.name() && event.hasValue()) { + try { + var delta = parseFunction.apply(parserConfig, event); + delta.forEachRemaining(results::offer); + } catch (Exception e) { + logger.warn("Failed to parse event from inference provider: {}", event); + throw e; + } + } + } + + return results; + } + + @FunctionalInterface + public interface ParseChunkFunction { + Iterator apply(XContentParserConfiguration parserConfig, ServerSentEvent event) throws IOException; + } + @Override public void subscribe(Flow.Subscriber subscriber) { if (downstream != null) { @@ -51,7 +85,7 @@ public void request(long n) { if (isClosed.get()) { downstream.onComplete(); } else if (upstream != null) { - upstream.request(n); + upstreamRequest(n); } else { pendingRequests.accumulateAndGet(n, Long::sum); } @@ -67,6 +101,13 @@ public void cancel() { }; } + /** + * Guaranteed to be called when the upstream is set and this processor had not been closed. + */ + protected void upstreamRequest(long n) { + upstream.request(n); + } + protected void onCancel() {} @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableAction.java index 4e97554b56445..b43e5ab70e2f2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableAction.java @@ -12,7 +12,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.external.http.sender.RequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; @@ -34,13 +33,7 @@ public SingleInputSenderExecutableAction( @Override public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener listener) { - if (inferenceInputs instanceof DocumentsOnlyInput == false) { - listener.onFailure(new ElasticsearchStatusException("Invalid inference input type", RestStatus.INTERNAL_SERVER_ERROR)); - return; - } - - var docsOnlyInput = (DocumentsOnlyInput) inferenceInputs; - if (docsOnlyInput.getInputs().size() > 1) { + if (inferenceInputs.inputSize() > 1) { listener.onFailure( new ElasticsearchStatusException(requestTypeForInputValidationError + " only accepts 1 input", RestStatus.BAD_REQUEST) ); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreator.java index 9c83264b5581f..bd5c53d589df0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreator.java @@ -26,7 +26,7 @@ * Provides a way to construct an {@link ExecutableAction} using the visitor pattern based on the openai model type. */ public class OpenAiActionCreator implements OpenAiActionVisitor { - private static final String COMPLETION_ERROR_PREFIX = "OpenAI chat completions"; + public static final String COMPLETION_ERROR_PREFIX = "OpenAI chat completions"; private final Sender sender; private final ServiceComponents serviceComponents; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchCompletionRequestManager.java index a0a44e62f9f73..e7a960f1316f2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchCompletionRequestManager.java @@ -69,7 +69,7 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - List input = DocumentsOnlyInput.of(inferenceInputs).getInputs(); + List input = inferenceInputs.castTo(ChatCompletionInput.class).getInputs(); AlibabaCloudSearchCompletionRequest request = new AlibabaCloudSearchCompletionRequest(account, input, model); execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java index 69a5c665feb86..3929585a0745d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java @@ -44,10 +44,10 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - var docsOnly = DocumentsOnlyInput.of(inferenceInputs); - var docsInput = docsOnly.getInputs(); - var stream = docsOnly.stream(); - var requestEntity = AmazonBedrockChatCompletionEntityFactory.createEntity(model, docsInput); + var chatCompletionInput = inferenceInputs.castTo(ChatCompletionInput.class); + var inputs = chatCompletionInput.getInputs(); + var stream = chatCompletionInput.stream(); + var requestEntity = AmazonBedrockChatCompletionEntityFactory.createEntity(model, inputs); var request = new AmazonBedrockChatCompletionRequest(model, requestEntity, timeout, stream); var responseHandler = new AmazonBedrockChatCompletionResponseHandler(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AnthropicCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AnthropicCompletionRequestManager.java index 5418b3dd9840b..6d4aeb9e31bac 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AnthropicCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AnthropicCompletionRequestManager.java @@ -46,10 +46,10 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - var docsOnly = DocumentsOnlyInput.of(inferenceInputs); - var docsInput = docsOnly.getInputs(); - var stream = docsOnly.stream(); - AnthropicChatCompletionRequest request = new AnthropicChatCompletionRequest(docsInput, model, stream); + var chatCompletionInput = inferenceInputs.castTo(ChatCompletionInput.class); + var inputs = chatCompletionInput.getInputs(); + var stream = chatCompletionInput.stream(); + AnthropicChatCompletionRequest request = new AnthropicChatCompletionRequest(inputs, model, stream); execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioChatCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioChatCompletionRequestManager.java index 21cec68b14a49..affd2e3a7760e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioChatCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioChatCompletionRequestManager.java @@ -41,10 +41,10 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - var docsOnly = DocumentsOnlyInput.of(inferenceInputs); - var docsInput = docsOnly.getInputs(); - var stream = docsOnly.stream(); - AzureAiStudioChatCompletionRequest request = new AzureAiStudioChatCompletionRequest(model, docsInput, stream); + var chatCompletionInput = inferenceInputs.castTo(ChatCompletionInput.class); + var inputs = chatCompletionInput.getInputs(); + var stream = chatCompletionInput.stream(); + AzureAiStudioChatCompletionRequest request = new AzureAiStudioChatCompletionRequest(model, inputs, stream); execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiCompletionRequestManager.java index d036559ec3dcb..c2f5f3e9db5ed 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiCompletionRequestManager.java @@ -46,10 +46,10 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - var docsOnly = DocumentsOnlyInput.of(inferenceInputs); - var docsInput = docsOnly.getInputs(); - var stream = docsOnly.stream(); - AzureOpenAiCompletionRequest request = new AzureOpenAiCompletionRequest(docsInput, model, stream); + var chatCompletionInput = inferenceInputs.castTo(ChatCompletionInput.class); + var inputs = chatCompletionInput.getInputs(); + var stream = chatCompletionInput.stream(); + AzureOpenAiCompletionRequest request = new AzureOpenAiCompletionRequest(inputs, model, stream); execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ChatCompletionInput.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ChatCompletionInput.java new file mode 100644 index 0000000000000..928da95d9c2f0 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ChatCompletionInput.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import java.util.List; +import java.util.Objects; + +/** + * This class encapsulates the input text passed by the request and indicates whether the response should be streamed. + * The main difference between this class and {@link UnifiedChatInput} is this should only be used for + * {@link org.elasticsearch.inference.TaskType#COMPLETION} originating through the + * {@link org.elasticsearch.inference.InferenceService#infer} code path. These are requests sent to the + * API without using the _unified route. + */ +public class ChatCompletionInput extends InferenceInputs { + private final List input; + + public ChatCompletionInput(List input) { + this(input, false); + } + + public ChatCompletionInput(List input, boolean stream) { + super(stream); + this.input = Objects.requireNonNull(input); + } + + public List getInputs() { + return this.input; + } + + public int inputSize() { + return input.size(); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java index ae46fbe0fef87..40cd03c87664e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java @@ -50,10 +50,10 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - var docsOnly = DocumentsOnlyInput.of(inferenceInputs); - var docsInput = docsOnly.getInputs(); - var stream = docsOnly.stream(); - CohereCompletionRequest request = new CohereCompletionRequest(docsInput, model, stream); + var chatCompletionInput = inferenceInputs.castTo(ChatCompletionInput.class); + var inputs = chatCompletionInput.getInputs(); + var stream = chatCompletionInput.stream(); + CohereCompletionRequest request = new CohereCompletionRequest(inputs, model, stream); execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/DocumentsOnlyInput.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/DocumentsOnlyInput.java index 8cf411d84c932..3feb79d3de6cc 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/DocumentsOnlyInput.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/DocumentsOnlyInput.java @@ -14,30 +14,28 @@ public class DocumentsOnlyInput extends InferenceInputs { public static DocumentsOnlyInput of(InferenceInputs inferenceInputs) { if (inferenceInputs instanceof DocumentsOnlyInput == false) { - throw createUnsupportedTypeException(inferenceInputs); + throw createUnsupportedTypeException(inferenceInputs, DocumentsOnlyInput.class); } return (DocumentsOnlyInput) inferenceInputs; } private final List input; - private final boolean stream; public DocumentsOnlyInput(List input) { this(input, false); } public DocumentsOnlyInput(List input, boolean stream) { - super(); + super(stream); this.input = Objects.requireNonNull(input); - this.stream = stream; } public List getInputs() { return this.input; } - public boolean stream() { - return stream; + public int inputSize() { + return input.size(); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java index abe50c6fae3f9..0097f9c08ea21 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java @@ -51,7 +51,10 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - GoogleAiStudioCompletionRequest request = new GoogleAiStudioCompletionRequest(DocumentsOnlyInput.of(inferenceInputs), model); + GoogleAiStudioCompletionRequest request = new GoogleAiStudioCompletionRequest( + inferenceInputs.castTo(ChatCompletionInput.class), + model + ); execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceInputs.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceInputs.java index dd241857ef0c4..e85ea6f1d9b35 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceInputs.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceInputs.java @@ -10,7 +10,29 @@ import org.elasticsearch.common.Strings; public abstract class InferenceInputs { - public static IllegalArgumentException createUnsupportedTypeException(InferenceInputs inferenceInputs) { - return new IllegalArgumentException(Strings.format("Unsupported inference inputs type: [%s]", inferenceInputs.getClass())); + private final boolean stream; + + public InferenceInputs(boolean stream) { + this.stream = stream; + } + + public static IllegalArgumentException createUnsupportedTypeException(InferenceInputs inferenceInputs, Class clazz) { + return new IllegalArgumentException( + Strings.format("Unable to convert inference inputs type: [%s] to [%s]", inferenceInputs.getClass(), clazz) + ); } + + public T castTo(Class clazz) { + if (clazz.isInstance(this) == false) { + throw createUnsupportedTypeException(this, clazz); + } + + return clazz.cast(this); + } + + public boolean stream() { + return stream; + } + + public abstract int inputSize(); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java index cea89332e5bf0..4d730be6aa6bd 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java @@ -15,7 +15,7 @@ import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; import org.elasticsearch.xpack.inference.external.openai.OpenAiChatCompletionResponseHandler; -import org.elasticsearch.xpack.inference.external.request.openai.OpenAiChatCompletionRequest; +import org.elasticsearch.xpack.inference.external.request.openai.OpenAiUnifiedChatCompletionRequest; import org.elasticsearch.xpack.inference.external.response.openai.OpenAiChatCompletionResponseEntity; import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel; @@ -25,8 +25,8 @@ public class OpenAiCompletionRequestManager extends OpenAiRequestManager { private static final Logger logger = LogManager.getLogger(OpenAiCompletionRequestManager.class); - private static final ResponseHandler HANDLER = createCompletionHandler(); + static final String USER_ROLE = "user"; public static OpenAiCompletionRequestManager of(OpenAiChatCompletionModel model, ThreadPool threadPool) { return new OpenAiCompletionRequestManager(Objects.requireNonNull(model), Objects.requireNonNull(threadPool)); @@ -35,7 +35,7 @@ public static OpenAiCompletionRequestManager of(OpenAiChatCompletionModel model, private final OpenAiChatCompletionModel model; private OpenAiCompletionRequestManager(OpenAiChatCompletionModel model, ThreadPool threadPool) { - super(threadPool, model, OpenAiChatCompletionRequest::buildDefaultUri); + super(threadPool, model, OpenAiUnifiedChatCompletionRequest::buildDefaultUri); this.model = Objects.requireNonNull(model); } @@ -46,10 +46,8 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - var docsOnly = DocumentsOnlyInput.of(inferenceInputs); - var docsInput = docsOnly.getInputs(); - var stream = docsOnly.stream(); - OpenAiChatCompletionRequest request = new OpenAiChatCompletionRequest(docsInput, model, stream); + var chatCompletionInputs = inferenceInputs.castTo(ChatCompletionInput.class); + var request = new OpenAiUnifiedChatCompletionRequest(new UnifiedChatInput(chatCompletionInputs, USER_ROLE), model); execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiUnifiedCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiUnifiedCompletionRequestManager.java new file mode 100644 index 0000000000000..3b0f770e3e061 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiUnifiedCompletionRequestManager.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; +import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; +import org.elasticsearch.xpack.inference.external.openai.OpenAiUnifiedChatCompletionResponseHandler; +import org.elasticsearch.xpack.inference.external.request.openai.OpenAiUnifiedChatCompletionRequest; +import org.elasticsearch.xpack.inference.external.response.openai.OpenAiChatCompletionResponseEntity; +import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel; + +import java.util.Objects; +import java.util.function.Supplier; + +public class OpenAiUnifiedCompletionRequestManager extends OpenAiRequestManager { + + private static final Logger logger = LogManager.getLogger(OpenAiUnifiedCompletionRequestManager.class); + + private static final ResponseHandler HANDLER = createCompletionHandler(); + + public static OpenAiUnifiedCompletionRequestManager of(OpenAiChatCompletionModel model, ThreadPool threadPool) { + return new OpenAiUnifiedCompletionRequestManager(Objects.requireNonNull(model), Objects.requireNonNull(threadPool)); + } + + private final OpenAiChatCompletionModel model; + + private OpenAiUnifiedCompletionRequestManager(OpenAiChatCompletionModel model, ThreadPool threadPool) { + super(threadPool, model, OpenAiUnifiedChatCompletionRequest::buildDefaultUri); + this.model = Objects.requireNonNull(model); + } + + @Override + public void execute( + InferenceInputs inferenceInputs, + RequestSender requestSender, + Supplier hasRequestCompletedFunction, + ActionListener listener + ) { + + OpenAiUnifiedChatCompletionRequest request = new OpenAiUnifiedChatCompletionRequest( + inferenceInputs.castTo(UnifiedChatInput.class), + model + ); + + execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); + } + + private static ResponseHandler createCompletionHandler() { + return new OpenAiUnifiedChatCompletionResponseHandler("openai completion", OpenAiChatCompletionResponseEntity::fromResponse); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/QueryAndDocsInputs.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/QueryAndDocsInputs.java index 50bb77b307db3..5af5245ac5b40 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/QueryAndDocsInputs.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/QueryAndDocsInputs.java @@ -14,7 +14,7 @@ public class QueryAndDocsInputs extends InferenceInputs { public static QueryAndDocsInputs of(InferenceInputs inferenceInputs) { if (inferenceInputs instanceof QueryAndDocsInputs == false) { - throw createUnsupportedTypeException(inferenceInputs); + throw createUnsupportedTypeException(inferenceInputs, QueryAndDocsInputs.class); } return (QueryAndDocsInputs) inferenceInputs; @@ -22,17 +22,15 @@ public static QueryAndDocsInputs of(InferenceInputs inferenceInputs) { private final String query; private final List chunks; - private final boolean stream; public QueryAndDocsInputs(String query, List chunks) { this(query, chunks, false); } public QueryAndDocsInputs(String query, List chunks, boolean stream) { - super(); + super(stream); this.query = Objects.requireNonNull(query); this.chunks = Objects.requireNonNull(chunks); - this.stream = stream; } public String getQuery() { @@ -43,8 +41,7 @@ public List getChunks() { return chunks; } - public boolean stream() { - return stream; + public int inputSize() { + return chunks.size(); } - } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInput.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInput.java new file mode 100644 index 0000000000000..f89fa1ee37a6f --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInput.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.UnifiedCompletionRequest; + +import java.util.List; +import java.util.Objects; + +/** + * This class encapsulates the unified request. + * The main difference between this class and {@link ChatCompletionInput} is this should only be used for + * {@link org.elasticsearch.inference.TaskType#COMPLETION} originating through the + * {@link org.elasticsearch.inference.InferenceService#unifiedCompletionInfer(Model, UnifiedCompletionRequest, TimeValue, ActionListener)} + * code path. These are requests sent to the API with the _unified route. + */ +public class UnifiedChatInput extends InferenceInputs { + private final UnifiedCompletionRequest request; + + public UnifiedChatInput(UnifiedCompletionRequest request, boolean stream) { + super(stream); + this.request = Objects.requireNonNull(request); + } + + public UnifiedChatInput(ChatCompletionInput completionInput, String roleValue) { + this(completionInput.getInputs(), roleValue, completionInput.stream()); + } + + public UnifiedChatInput(List inputs, String roleValue, boolean stream) { + this(UnifiedCompletionRequest.of(convertToMessages(inputs, roleValue)), stream); + } + + private static List convertToMessages(List inputs, String roleValue) { + return inputs.stream() + .map( + value -> new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString(value), + roleValue, + null, + null, + null + ) + ) + .toList(); + } + + public UnifiedCompletionRequest getRequest() { + return request; + } + + public int inputSize() { + return request.messages().size(); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java index 6e006fe255956..48c8132035b50 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java @@ -18,10 +18,8 @@ import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; import org.elasticsearch.xpack.inference.common.DelegatingProcessor; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent; -import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventField; import java.io.IOException; -import java.util.ArrayDeque; import java.util.Collections; import java.util.Deque; import java.util.Iterator; @@ -115,19 +113,7 @@ public class OpenAiStreamingProcessor extends DelegatingProcessor item) throws Exception { var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); - - var results = new ArrayDeque(item.size()); - for (ServerSentEvent event : item) { - if (ServerSentEventField.DATA == event.name() && event.hasValue()) { - try { - var delta = parse(parserConfig, event); - delta.forEachRemaining(results::offer); - } catch (Exception e) { - log.warn("Failed to parse event from inference provider: {}", event); - throw e; - } - } - } + var results = parseEvent(item, OpenAiStreamingProcessor::parse, parserConfig, log); if (results.isEmpty()) { upstream().request(1); @@ -136,7 +122,7 @@ protected void next(Deque item) throws Exception { } } - private Iterator parse(XContentParserConfiguration parserConfig, ServerSentEvent event) + private static Iterator parse(XContentParserConfiguration parserConfig, ServerSentEvent event) throws IOException { if (DONE_MESSAGE.equalsIgnoreCase(event.value())) { return Collections.emptyIterator(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandler.java new file mode 100644 index 0000000000000..fce2556efc5e0 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedChatCompletionResponseHandler.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.openai; + +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xpack.core.inference.results.StreamingUnifiedChatCompletionResults; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor; + +import java.util.concurrent.Flow; + +public class OpenAiUnifiedChatCompletionResponseHandler extends OpenAiChatCompletionResponseHandler { + public OpenAiUnifiedChatCompletionResponseHandler(String requestType, ResponseParser parseFunction) { + super(requestType, parseFunction); + } + + @Override + public InferenceServiceResults parseResult(Request request, Flow.Publisher flow) { + var serverSentEventProcessor = new ServerSentEventProcessor(new ServerSentEventParser()); + var openAiProcessor = new OpenAiUnifiedStreamingProcessor(); + + flow.subscribe(serverSentEventProcessor); + serverSentEventProcessor.subscribe(openAiProcessor); + return new StreamingUnifiedChatCompletionResults(openAiProcessor); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessor.java new file mode 100644 index 0000000000000..599d71df3dcfa --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessor.java @@ -0,0 +1,287 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.openai; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.results.StreamingUnifiedChatCompletionResults; +import org.elasticsearch.xpack.inference.common.DelegatingProcessor; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.LinkedBlockingDeque; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken; + +public class OpenAiUnifiedStreamingProcessor extends DelegatingProcessor, ChunkedToXContent> { + public static final String FUNCTION_FIELD = "function"; + private static final Logger logger = LogManager.getLogger(OpenAiUnifiedStreamingProcessor.class); + + private static final String CHOICES_FIELD = "choices"; + private static final String DELTA_FIELD = "delta"; + private static final String CONTENT_FIELD = "content"; + private static final String DONE_MESSAGE = "[done]"; + private static final String REFUSAL_FIELD = "refusal"; + private static final String TOOL_CALLS_FIELD = "tool_calls"; + public static final String ROLE_FIELD = "role"; + public static final String FINISH_REASON_FIELD = "finish_reason"; + public static final String INDEX_FIELD = "index"; + public static final String OBJECT_FIELD = "object"; + public static final String MODEL_FIELD = "model"; + public static final String ID_FIELD = "id"; + public static final String CHOICE_FIELD = "choice"; + public static final String USAGE_FIELD = "usage"; + public static final String TYPE_FIELD = "type"; + public static final String NAME_FIELD = "name"; + public static final String ARGUMENTS_FIELD = "arguments"; + public static final String COMPLETION_TOKENS_FIELD = "completion_tokens"; + public static final String PROMPT_TOKENS_FIELD = "prompt_tokens"; + public static final String TOTAL_TOKENS_FIELD = "total_tokens"; + + private final Deque buffer = new LinkedBlockingDeque<>(); + + @Override + protected void upstreamRequest(long n) { + if (buffer.isEmpty()) { + super.upstreamRequest(n); + } else { + downstream().onNext(new StreamingUnifiedChatCompletionResults.Results(singleItem(buffer.poll()))); + } + } + + @Override + protected void next(Deque item) throws Exception { + var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); + var results = parseEvent(item, OpenAiUnifiedStreamingProcessor::parse, parserConfig, logger); + + if (results.isEmpty()) { + upstream().request(1); + } else if (results.size() == 1) { + downstream().onNext(new StreamingUnifiedChatCompletionResults.Results(results)); + } else { + // results > 1, but openai spec only wants 1 chunk per SSE event + var firstItem = singleItem(results.poll()); + while (results.isEmpty() == false) { + buffer.offer(results.poll()); + } + downstream().onNext(new StreamingUnifiedChatCompletionResults.Results(firstItem)); + } + } + + private static Iterator parse( + XContentParserConfiguration parserConfig, + ServerSentEvent event + ) throws IOException { + if (DONE_MESSAGE.equalsIgnoreCase(event.value())) { + return Collections.emptyIterator(); + } + + try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, event.value())) { + moveToFirstToken(jsonParser); + + XContentParser.Token token = jsonParser.currentToken(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); + + StreamingUnifiedChatCompletionResults.ChatCompletionChunk chunk = ChatCompletionChunkParser.parse(jsonParser); + + return Collections.singleton(chunk).iterator(); + } + } + + public static class ChatCompletionChunkParser { + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + "chat_completion_chunk", + true, + args -> new StreamingUnifiedChatCompletionResults.ChatCompletionChunk( + (String) args[0], + (List) args[1], + (String) args[2], + (String) args[3], + (StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Usage) args[4] + ) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField(ID_FIELD)); + PARSER.declareObjectArray( + ConstructingObjectParser.constructorArg(), + (p, c) -> ChatCompletionChunkParser.ChoiceParser.parse(p), + new ParseField(CHOICES_FIELD) + ); + PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField(MODEL_FIELD)); + PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField(OBJECT_FIELD)); + PARSER.declareObjectOrNull( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> ChatCompletionChunkParser.UsageParser.parse(p), + null, + new ParseField(USAGE_FIELD) + ); + } + + public static StreamingUnifiedChatCompletionResults.ChatCompletionChunk parse(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + private static class ChoiceParser { + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + CHOICE_FIELD, + true, + args -> new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice( + (StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta) args[0], + (String) args[1], + (int) args[2] + ) + ); + + static { + PARSER.declareObject( + ConstructingObjectParser.constructorArg(), + (p, c) -> ChatCompletionChunkParser.DeltaParser.parse(p), + new ParseField(DELTA_FIELD) + ); + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), new ParseField(FINISH_REASON_FIELD)); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), new ParseField(INDEX_FIELD)); + } + + public static StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + } + + private static class DeltaParser { + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser< + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta, + Void> PARSER = new ConstructingObjectParser<>( + DELTA_FIELD, + true, + args -> new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta( + (String) args[0], + (String) args[1], + (String) args[2], + (List) args[3] + ) + ); + + static { + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), new ParseField(CONTENT_FIELD)); + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), new ParseField(REFUSAL_FIELD)); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField(ROLE_FIELD)); + PARSER.declareObjectArray( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> ChatCompletionChunkParser.ToolCallParser.parse(p), + new ParseField(TOOL_CALLS_FIELD) + ); + } + + public static StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta parse(XContentParser parser) + throws IOException { + return PARSER.parse(parser, null); + } + } + + private static class ToolCallParser { + private static final ConstructingObjectParser< + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall, + Void> PARSER = new ConstructingObjectParser<>( + "tool_call", + true, + args -> new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall( + (int) args[0], + (String) args[1], + (StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall.Function) args[2], + (String) args[3] + ) + ); + + static { + PARSER.declareInt(ConstructingObjectParser.constructorArg(), new ParseField(INDEX_FIELD)); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField(ID_FIELD)); + PARSER.declareObject( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> ChatCompletionChunkParser.FunctionParser.parse(p), + new ParseField(FUNCTION_FIELD) + ); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField(TYPE_FIELD)); + } + + public static StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall parse(XContentParser parser) + throws IOException { + return PARSER.parse(parser, null); + } + } + + private static class FunctionParser { + private static final ConstructingObjectParser< + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall.Function, + Void> PARSER = new ConstructingObjectParser<>( + FUNCTION_FIELD, + true, + args -> new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall.Function( + (String) args[0], + (String) args[1] + ) + ); + + static { + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField(ARGUMENTS_FIELD)); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField(NAME_FIELD)); + } + + public static StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall.Function parse( + XContentParser parser + ) throws IOException { + return PARSER.parse(parser, null); + } + } + + private static class UsageParser { + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + USAGE_FIELD, + true, + args -> new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Usage((int) args[0], (int) args[1], (int) args[2]) + ); + + static { + PARSER.declareInt(ConstructingObjectParser.constructorArg(), new ParseField(COMPLETION_TOKENS_FIELD)); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), new ParseField(PROMPT_TOKENS_FIELD)); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), new ParseField(TOTAL_TOKENS_FIELD)); + } + + public static StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Usage parse(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + } + } + + private Deque singleItem( + StreamingUnifiedChatCompletionResults.ChatCompletionChunk result + ) { + var deque = new ArrayDeque(1); + deque.offer(result); + return deque; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioCompletionRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioCompletionRequest.java index 80770d63ef139..b1af18d03dda4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioCompletionRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioCompletionRequest.java @@ -14,7 +14,7 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.request.HttpRequest; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel; @@ -27,13 +27,13 @@ public class GoogleAiStudioCompletionRequest implements GoogleAiStudioRequest { private static final String ALT_PARAM = "alt"; private static final String SSE_VALUE = "sse"; - private final DocumentsOnlyInput input; + private final ChatCompletionInput input; private final LazyInitializable uri; private final GoogleAiStudioCompletionModel model; - public GoogleAiStudioCompletionRequest(DocumentsOnlyInput input, GoogleAiStudioCompletionModel model) { + public GoogleAiStudioCompletionRequest(ChatCompletionInput input, GoogleAiStudioCompletionModel model) { this.input = Objects.requireNonNull(input); this.model = Objects.requireNonNull(model); this.uri = new LazyInitializable<>(() -> model.uri(input.stream())); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntity.java deleted file mode 100644 index 867a7ca80cbcb..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntity.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.openai; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.List; -import java.util.Objects; - -public class OpenAiChatCompletionRequestEntity implements ToXContentObject { - - private static final String MESSAGES_FIELD = "messages"; - private static final String MODEL_FIELD = "model"; - - private static final String NUMBER_OF_RETURNED_CHOICES_FIELD = "n"; - - private static final String ROLE_FIELD = "role"; - private static final String USER_FIELD = "user"; - private static final String CONTENT_FIELD = "content"; - private static final String STREAM_FIELD = "stream"; - - private final List messages; - private final String model; - - private final String user; - private final boolean stream; - - public OpenAiChatCompletionRequestEntity(List messages, String model, String user, boolean stream) { - Objects.requireNonNull(messages); - Objects.requireNonNull(model); - - this.messages = messages; - this.model = model; - this.user = user; - this.stream = stream; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.startArray(MESSAGES_FIELD); - { - for (String message : messages) { - builder.startObject(); - - { - builder.field(ROLE_FIELD, USER_FIELD); - builder.field(CONTENT_FIELD, message); - } - - builder.endObject(); - } - } - builder.endArray(); - - builder.field(MODEL_FIELD, model); - builder.field(NUMBER_OF_RETURNED_CHOICES_FIELD, 1); - - if (Strings.isNullOrEmpty(user) == false) { - builder.field(USER_FIELD, user); - } - - if (stream) { - builder.field(STREAM_FIELD, true); - } - - builder.endObject(); - - return builder; - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequest.java similarity index 80% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequest.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequest.java index 99a025e70d003..2e6bdb748fd33 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequest.java @@ -13,6 +13,7 @@ import org.apache.http.entity.ByteArrayEntity; import org.elasticsearch.common.Strings; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.external.openai.OpenAiAccount; import org.elasticsearch.xpack.inference.external.request.HttpRequest; import org.elasticsearch.xpack.inference.external.request.Request; @@ -21,24 +22,21 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; -import java.util.List; import java.util.Objects; import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader; import static org.elasticsearch.xpack.inference.external.request.openai.OpenAiUtils.createOrgHeader; -public class OpenAiChatCompletionRequest implements OpenAiRequest { +public class OpenAiUnifiedChatCompletionRequest implements OpenAiRequest { private final OpenAiAccount account; - private final List input; private final OpenAiChatCompletionModel model; - private final boolean stream; + private final UnifiedChatInput unifiedChatInput; - public OpenAiChatCompletionRequest(List input, OpenAiChatCompletionModel model, boolean stream) { - this.account = OpenAiAccount.of(model, OpenAiChatCompletionRequest::buildDefaultUri); - this.input = Objects.requireNonNull(input); + public OpenAiUnifiedChatCompletionRequest(UnifiedChatInput unifiedChatInput, OpenAiChatCompletionModel model) { + this.account = OpenAiAccount.of(model, OpenAiUnifiedChatCompletionRequest::buildDefaultUri); + this.unifiedChatInput = Objects.requireNonNull(unifiedChatInput); this.model = Objects.requireNonNull(model); - this.stream = stream; } @Override @@ -46,9 +44,7 @@ public HttpRequest createHttpRequest() { HttpPost httpPost = new HttpPost(account.uri()); ByteArrayEntity byteEntity = new ByteArrayEntity( - Strings.toString( - new OpenAiChatCompletionRequestEntity(input, model.getServiceSettings().modelId(), model.getTaskSettings().user(), stream) - ).getBytes(StandardCharsets.UTF_8) + Strings.toString(new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model)).getBytes(StandardCharsets.UTF_8) ); httpPost.setEntity(byteEntity); @@ -87,7 +83,7 @@ public String getInferenceEntityId() { @Override public boolean isStreaming() { - return stream; + return unifiedChatInput.stream(); } public static URI buildDefaultUri() throws URISyntaxException { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntity.java new file mode 100644 index 0000000000000..50339bf851f7d --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntity.java @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.openai; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.inference.UnifiedCompletionRequest; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; +import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel; + +import java.io.IOException; +import java.util.Objects; + +public class OpenAiUnifiedChatCompletionRequestEntity implements ToXContentObject { + + public static final String NAME_FIELD = "name"; + public static final String TOOL_CALL_ID_FIELD = "tool_call_id"; + public static final String TOOL_CALLS_FIELD = "tool_calls"; + public static final String ID_FIELD = "id"; + public static final String FUNCTION_FIELD = "function"; + public static final String ARGUMENTS_FIELD = "arguments"; + public static final String DESCRIPTION_FIELD = "description"; + public static final String PARAMETERS_FIELD = "parameters"; + public static final String STRICT_FIELD = "strict"; + public static final String TOP_P_FIELD = "top_p"; + public static final String USER_FIELD = "user"; + public static final String STREAM_FIELD = "stream"; + private static final String NUMBER_OF_RETURNED_CHOICES_FIELD = "n"; + private static final String MODEL_FIELD = "model"; + public static final String MESSAGES_FIELD = "messages"; + private static final String ROLE_FIELD = "role"; + private static final String CONTENT_FIELD = "content"; + private static final String MAX_COMPLETION_TOKENS_FIELD = "max_completion_tokens"; + private static final String STOP_FIELD = "stop"; + private static final String TEMPERATURE_FIELD = "temperature"; + private static final String TOOL_CHOICE_FIELD = "tool_choice"; + private static final String TOOL_FIELD = "tools"; + private static final String TEXT_FIELD = "text"; + private static final String TYPE_FIELD = "type"; + private static final String STREAM_OPTIONS_FIELD = "stream_options"; + private static final String INCLUDE_USAGE_FIELD = "include_usage"; + + private final UnifiedCompletionRequest unifiedRequest; + private final boolean stream; + private final OpenAiChatCompletionModel model; + + public OpenAiUnifiedChatCompletionRequestEntity(UnifiedChatInput unifiedChatInput, OpenAiChatCompletionModel model) { + Objects.requireNonNull(unifiedChatInput); + + this.unifiedRequest = unifiedChatInput.getRequest(); + this.stream = unifiedChatInput.stream(); + this.model = Objects.requireNonNull(model); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startArray(MESSAGES_FIELD); + { + for (UnifiedCompletionRequest.Message message : unifiedRequest.messages()) { + builder.startObject(); + { + switch (message.content()) { + case UnifiedCompletionRequest.ContentString contentString -> builder.field(CONTENT_FIELD, contentString.content()); + case UnifiedCompletionRequest.ContentObjects contentObjects -> { + builder.startArray(CONTENT_FIELD); + for (UnifiedCompletionRequest.ContentObject contentObject : contentObjects.contentObjects()) { + builder.startObject(); + builder.field(TEXT_FIELD, contentObject.text()); + builder.field(TYPE_FIELD, contentObject.type()); + builder.endObject(); + } + builder.endArray(); + } + } + + builder.field(ROLE_FIELD, message.role()); + if (message.name() != null) { + builder.field(NAME_FIELD, message.name()); + } + if (message.toolCallId() != null) { + builder.field(TOOL_CALL_ID_FIELD, message.toolCallId()); + } + if (message.toolCalls() != null) { + builder.startArray(TOOL_CALLS_FIELD); + for (UnifiedCompletionRequest.ToolCall toolCall : message.toolCalls()) { + builder.startObject(); + { + builder.field(ID_FIELD, toolCall.id()); + builder.startObject(FUNCTION_FIELD); + { + builder.field(ARGUMENTS_FIELD, toolCall.function().arguments()); + builder.field(NAME_FIELD, toolCall.function().name()); + } + builder.endObject(); + builder.field(TYPE_FIELD, toolCall.type()); + } + builder.endObject(); + } + builder.endArray(); + } + } + builder.endObject(); + } + } + builder.endArray(); + + builder.field(MODEL_FIELD, model.getServiceSettings().modelId()); + if (unifiedRequest.maxCompletionTokens() != null) { + builder.field(MAX_COMPLETION_TOKENS_FIELD, unifiedRequest.maxCompletionTokens()); + } + + builder.field(NUMBER_OF_RETURNED_CHOICES_FIELD, 1); + + if (unifiedRequest.stop() != null && unifiedRequest.stop().isEmpty() == false) { + builder.field(STOP_FIELD, unifiedRequest.stop()); + } + if (unifiedRequest.temperature() != null) { + builder.field(TEMPERATURE_FIELD, unifiedRequest.temperature()); + } + if (unifiedRequest.toolChoice() != null) { + if (unifiedRequest.toolChoice() instanceof UnifiedCompletionRequest.ToolChoiceString) { + builder.field(TOOL_CHOICE_FIELD, ((UnifiedCompletionRequest.ToolChoiceString) unifiedRequest.toolChoice()).value()); + } else if (unifiedRequest.toolChoice() instanceof UnifiedCompletionRequest.ToolChoiceObject) { + builder.startObject(TOOL_CHOICE_FIELD); + { + builder.field(TYPE_FIELD, ((UnifiedCompletionRequest.ToolChoiceObject) unifiedRequest.toolChoice()).type()); + builder.startObject(FUNCTION_FIELD); + { + builder.field( + NAME_FIELD, + ((UnifiedCompletionRequest.ToolChoiceObject) unifiedRequest.toolChoice()).function().name() + ); + } + builder.endObject(); + } + builder.endObject(); + } + } + if (unifiedRequest.tools() != null && unifiedRequest.tools().isEmpty() == false) { + builder.startArray(TOOL_FIELD); + for (UnifiedCompletionRequest.Tool t : unifiedRequest.tools()) { + builder.startObject(); + { + builder.field(TYPE_FIELD, t.type()); + builder.startObject(FUNCTION_FIELD); + { + builder.field(DESCRIPTION_FIELD, t.function().description()); + builder.field(NAME_FIELD, t.function().name()); + builder.field(PARAMETERS_FIELD, t.function().parameters()); + if (t.function().strict() != null) { + builder.field(STRICT_FIELD, t.function().strict()); + } + } + builder.endObject(); + } + builder.endObject(); + } + builder.endArray(); + } + if (unifiedRequest.topP() != null) { + builder.field(TOP_P_FIELD, unifiedRequest.topP()); + } + + if (Strings.isNullOrEmpty(model.getTaskSettings().user()) == false) { + builder.field(USER_FIELD, model.getTaskSettings().user()); + } + + builder.field(STREAM_FIELD, stream); + if (stream) { + builder.startObject(STREAM_OPTIONS_FIELD); + builder.field(INCLUDE_USAGE_FIELD, true); + builder.endObject(); + } + builder.endObject(); + + return builder; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/BaseInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/BaseInferenceAction.java index e72e68052f648..d911158e82296 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/BaseInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/BaseInferenceAction.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; @@ -21,27 +22,32 @@ import static org.elasticsearch.xpack.inference.rest.Paths.TASK_TYPE_OR_INFERENCE_ID; abstract class BaseInferenceAction extends BaseRestHandler { - @Override - protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - String inferenceEntityId; - TaskType taskType; + static Params parseParams(RestRequest restRequest) { if (restRequest.hasParam(INFERENCE_ID)) { - inferenceEntityId = restRequest.param(INFERENCE_ID); - taskType = TaskType.fromStringOrStatusException(restRequest.param(TASK_TYPE_OR_INFERENCE_ID)); + var inferenceEntityId = restRequest.param(INFERENCE_ID); + var taskType = TaskType.fromStringOrStatusException(restRequest.param(TASK_TYPE_OR_INFERENCE_ID)); + return new Params(inferenceEntityId, taskType); } else { - inferenceEntityId = restRequest.param(TASK_TYPE_OR_INFERENCE_ID); - taskType = TaskType.ANY; + return new Params(restRequest.param(TASK_TYPE_OR_INFERENCE_ID), TaskType.ANY); } + } + + record Params(String inferenceEntityId, TaskType taskType) {} + + static TimeValue parseTimeout(RestRequest restRequest) { + return restRequest.paramAsTime(InferenceAction.Request.TIMEOUT.getPreferredName(), InferenceAction.Request.DEFAULT_TIMEOUT); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + var params = parseParams(restRequest); InferenceAction.Request.Builder requestBuilder; try (var parser = restRequest.contentParser()) { - requestBuilder = InferenceAction.Request.parseRequest(inferenceEntityId, taskType, parser); + requestBuilder = InferenceAction.Request.parseRequest(params.inferenceEntityId(), params.taskType(), parser); } - var inferTimeout = restRequest.paramAsTime( - InferenceAction.Request.TIMEOUT.getPreferredName(), - InferenceAction.Request.DEFAULT_TIMEOUT - ); + var inferTimeout = parseTimeout(restRequest); requestBuilder.setInferenceTimeout(inferTimeout); var request = prepareInferenceRequest(requestBuilder); return channel -> client.execute(InferenceAction.INSTANCE, request, listener(channel)); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java index 55d6443b43c03..c46f211bb26af 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java @@ -30,6 +30,12 @@ public final class Paths { + "}/{" + INFERENCE_ID + "}/_stream"; + static final String UNIFIED_INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}/_unified"; + static final String UNIFIED_TASK_TYPE_INFERENCE_ID_PATH = "_inference/{" + + TASK_TYPE_OR_INFERENCE_ID + + "}/{" + + INFERENCE_ID + + "}/_unified"; private Paths() { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceAction.java new file mode 100644 index 0000000000000..5c71b560a6b9d --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceAction.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.rest; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.xpack.inference.rest.Paths.UNIFIED_INFERENCE_ID_PATH; +import static org.elasticsearch.xpack.inference.rest.Paths.UNIFIED_TASK_TYPE_INFERENCE_ID_PATH; + +@ServerlessScope(Scope.PUBLIC) +public class RestUnifiedCompletionInferenceAction extends BaseRestHandler { + @Override + public String getName() { + return "unified_inference_action"; + } + + @Override + public List routes() { + return List.of(new Route(POST, UNIFIED_INFERENCE_ID_PATH), new Route(POST, UNIFIED_TASK_TYPE_INFERENCE_ID_PATH)); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + var params = BaseInferenceAction.parseParams(restRequest); + + var inferTimeout = BaseInferenceAction.parseTimeout(restRequest); + + UnifiedCompletionAction.Request request; + try (var parser = restRequest.contentParser()) { + request = UnifiedCompletionAction.Request.parseRequest(params.inferenceEntityId(), params.taskType(), inferTimeout, parser); + } + + return channel -> client.execute(UnifiedCompletionAction.INSTANCE, request, new ServerSentEventsRestActionListener(channel)); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java index 8e2dac1ef9db2..e9b75e9ec7796 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java @@ -7,9 +7,11 @@ package org.elasticsearch.xpack.inference.services; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.InferenceService; @@ -17,11 +19,15 @@ import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnifiedCompletionRequest; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.external.http.sender.QueryAndDocsInputs; import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import java.io.IOException; import java.util.EnumSet; @@ -61,11 +67,31 @@ public void infer( ActionListener listener ) { init(); - if (query != null) { - doInfer(model, new QueryAndDocsInputs(query, input, stream), taskSettings, inputType, timeout, listener); - } else { - doInfer(model, new DocumentsOnlyInput(input, stream), taskSettings, inputType, timeout, listener); - } + var inferenceInput = createInput(model, input, query, stream); + doInfer(model, inferenceInput, taskSettings, inputType, timeout, listener); + } + + private static InferenceInputs createInput(Model model, List input, @Nullable String query, boolean stream) { + return switch (model.getTaskType()) { + case COMPLETION -> new ChatCompletionInput(input, stream); + case RERANK -> new QueryAndDocsInputs(query, input, stream); + case TEXT_EMBEDDING, SPARSE_EMBEDDING -> new DocumentsOnlyInput(input, stream); + default -> throw new ElasticsearchStatusException( + Strings.format("Invalid task type received when determining input type: [%s]", model.getTaskType().toString()), + RestStatus.BAD_REQUEST + ); + }; + } + + @Override + public void unifiedCompletionInfer( + Model model, + UnifiedCompletionRequest request, + TimeValue timeout, + ActionListener listener + ) { + init(); + doUnifiedCompletionInfer(model, new UnifiedChatInput(request, true), timeout, listener); } @Override @@ -92,6 +118,13 @@ protected abstract void doInfer( ActionListener listener ); + protected abstract void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ); + protected abstract void doChunkedInfer( Model model, DocumentsOnlyInput inputs, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java index ec4b8d9bb4d3d..7d05bac363fb1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java @@ -776,5 +776,9 @@ public static T nonNullOrDefault(@Nullable T requestValue, @Nullable T origi return requestValue == null ? originalSettingsValue : requestValue; } + public static void throwUnsupportedUnifiedCompletionOperation(String serviceName) { + throw new UnsupportedOperationException(Strings.format("The %s service does not support unified completion", serviceName)); + } + private ServiceUtils() {} } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java index 5adc2a11b19d9..ffd26b9ac534d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchUtils; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; @@ -57,14 +58,13 @@ import java.util.Map; import java.util.stream.Stream; -import static org.elasticsearch.inference.TaskType.SPARSE_EMBEDDING; -import static org.elasticsearch.inference.TaskType.TEXT_EMBEDDING; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceFields.EMBEDDING_MAX_BATCH_SIZE; import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettings.HOST; import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettings.HTTP_SCHEMA_NAME; @@ -261,6 +261,16 @@ public AlibabaCloudSearchModel parsePersistedConfig(String inferenceEntityId, Ta ); } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override public void doInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java index 48b3c3df03e11..d224e50bb650d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java @@ -40,6 +40,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -64,6 +65,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.MODEL_FIELD; import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.PROVIDER_FIELD; import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.REGION_FIELD; @@ -89,6 +91,16 @@ public AmazonBedrockService( this.amazonBedrockSender = amazonBedrockFactory.createSender(); } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override protected void doInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java index b3d503de8e3eb..f1840af18779f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -52,6 +53,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; public class AnthropicService extends SenderService { public static final String NAME = "anthropic"; @@ -192,6 +194,16 @@ public EnumSet supportedTaskTypes() { return supportedTaskTypes; } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override public void doInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java index bba331fc0b5df..f8ea11e4b15a5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java @@ -38,6 +38,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -63,6 +64,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.ENDPOINT_TYPE_FIELD; import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.PROVIDER_FIELD; import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.TARGET_FIELD; @@ -81,6 +83,16 @@ public AzureAiStudioService(HttpRequestSender.Factory factory, ServiceComponents super(factory, serviceComponents); } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override protected void doInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java index 16c94dfa9ad94..a38c265d2613c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -58,6 +59,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; import static org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiServiceFields.API_VERSION; import static org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiServiceFields.DEPLOYMENT_ID; import static org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiServiceFields.RESOURCE_NAME; @@ -233,6 +235,16 @@ public EnumSet supportedTaskTypes() { return supportedTaskTypes; } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override protected void doInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index b3d8b3b6efce3..ccb8d79dacd6c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -58,6 +59,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceFields.EMBEDDING_MAX_BATCH_SIZE; public class CohereService extends SenderService { @@ -232,6 +234,16 @@ public EnumSet supportedTaskTypes() { return supportedTaskTypes; } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override public void doInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java index b256861e7dd27..fe8ee52eb8816 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java @@ -38,6 +38,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -57,6 +58,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; public class ElasticInferenceService extends SenderService { @@ -76,6 +78,16 @@ public ElasticInferenceService( this.elasticInferenceServiceComponents = elasticInferenceServiceComponents; } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override protected void doInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index 0e64842f873d3..5f613d6be5869 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -31,6 +31,7 @@ import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnifiedCompletionRequest; import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; @@ -77,6 +78,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.MODEL_ID; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_THREADS; @@ -578,6 +580,16 @@ private static CustomElandEmbeddingModel updateModelWithEmbeddingDetails(CustomE ); } + @Override + public void unifiedCompletionInfer( + Model model, + UnifiedCompletionRequest request, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override public void infer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java index 57a8a66a3f3a6..b681722a82136 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java @@ -39,6 +39,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -64,6 +65,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; import static org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioServiceFields.EMBEDDING_MAX_BATCH_SIZE; public class GoogleAiStudioService extends SenderService { @@ -282,9 +284,8 @@ protected void doInfer( ) { if (model instanceof GoogleAiStudioCompletionModel completionModel) { var requestManager = new GoogleAiStudioCompletionRequestManager(completionModel, getServiceComponents().threadPool()); - var docsOnly = DocumentsOnlyInput.of(inputs); var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( - completionModel.uri(docsOnly.stream()), + completionModel.uri(inputs.stream()), "Google AI Studio completion" ); var action = new SingleInputSenderExecutableAction( @@ -308,6 +309,16 @@ protected void doInfer( } } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override protected void doChunkedInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java index 857d475499aae..87a2d98dca92c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -57,6 +58,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; import static org.elasticsearch.xpack.inference.services.googlevertexai.GoogleVertexAiServiceFields.EMBEDDING_MAX_BATCH_SIZE; import static org.elasticsearch.xpack.inference.services.googlevertexai.GoogleVertexAiServiceFields.LOCATION; import static org.elasticsearch.xpack.inference.services.googlevertexai.GoogleVertexAiServiceFields.PROJECT_ID; @@ -206,6 +208,16 @@ protected void doInfer( action.execute(inputs, timeout, listener); } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override protected void doChunkedInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java index 51cca72f26054..b74ec01cd76e7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java @@ -18,6 +18,7 @@ import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; +import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.SettingsConfiguration; @@ -31,6 +32,7 @@ import org.elasticsearch.xpack.inference.external.action.huggingface.HuggingFaceActionCreator; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.ServiceUtils; @@ -47,6 +49,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceFields.URL; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; public class HuggingFaceService extends HuggingFaceBaseService { public static final String NAME = "hugging_face"; @@ -139,6 +142,16 @@ protected void doChunkedInfer( } } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override public InferenceServiceConfiguration getConfiguration() { return Configuration.get(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java index 75920efa251f2..5b038781b96af 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceBaseService; @@ -49,6 +50,7 @@ import java.util.Map; import static org.elasticsearch.xpack.core.inference.results.ResultUtils.createInvalidChunkedResultException; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; import static org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserServiceSettings.URL; public class HuggingFaceElserService extends HuggingFaceBaseService { @@ -81,6 +83,16 @@ protected HuggingFaceModel createModel( }; } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override protected void doChunkedInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java index 981a3e95808ef..cc66d5fd7ee74 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -57,6 +58,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; import static org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserServiceSettings.URL; import static org.elasticsearch.xpack.inference.services.ibmwatsonx.IbmWatsonxServiceFields.API_VERSION; import static org.elasticsearch.xpack.inference.services.ibmwatsonx.IbmWatsonxServiceFields.EMBEDDING_MAX_BATCH_SIZE; @@ -276,6 +278,16 @@ protected void doInfer( action.execute(input, timeout, listener); } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override protected void doChunkedInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java index fe0edb851902b..881e7d36f2a21 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -58,6 +59,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; import static org.elasticsearch.xpack.inference.services.mistral.MistralConstants.MODEL_FIELD; public class MistralService extends SenderService { @@ -88,6 +90,16 @@ protected void doInfer( } } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + @Override protected void doChunkedInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java index 20ff1c617d21f..7b51b068708ca 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java @@ -32,10 +32,13 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; import org.elasticsearch.xpack.inference.external.action.openai.OpenAiActionCreator; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.OpenAiUnifiedCompletionRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -53,6 +56,8 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; +import static org.elasticsearch.xpack.inference.external.action.openai.OpenAiActionCreator.COMPLETION_ERROR_PREFIX; import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID; import static org.elasticsearch.xpack.inference.services.ServiceFields.URL; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; @@ -257,6 +262,28 @@ public void doInfer( action.execute(inputs, timeout, listener); } + @Override + public void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + if (model instanceof OpenAiChatCompletionModel == false) { + listener.onFailure(createInvalidModelException(model)); + return; + } + + OpenAiChatCompletionModel openAiModel = (OpenAiChatCompletionModel) model; + + var overriddenModel = OpenAiChatCompletionModel.of(openAiModel, inputs.getRequest()); + var requestCreator = OpenAiUnifiedCompletionRequestManager.of(overriddenModel, getServiceComponents().threadPool()); + var errorMessage = constructFailedToSendRequestMessage(overriddenModel.getServiceSettings().uri(), COMPLETION_ERROR_PREFIX); + var action = new SenderExecutableAction(getSender(), requestCreator, errorMessage); + + action.execute(inputs, timeout, listener); + } + @Override protected void doChunkedInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java index e721cd2955cf3..7d79d64b3a771 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java @@ -13,6 +13,7 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnifiedCompletionRequest; import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; @@ -24,6 +25,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import static org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields.USER; @@ -38,6 +40,26 @@ public static OpenAiChatCompletionModel of(OpenAiChatCompletionModel model, Map< return new OpenAiChatCompletionModel(model, OpenAiChatCompletionTaskSettings.of(model.getTaskSettings(), requestTaskSettings)); } + public static OpenAiChatCompletionModel of(OpenAiChatCompletionModel model, UnifiedCompletionRequest request) { + var originalModelServiceSettings = model.getServiceSettings(); + var overriddenServiceSettings = new OpenAiChatCompletionServiceSettings( + Objects.requireNonNullElse(request.model(), originalModelServiceSettings.modelId()), + originalModelServiceSettings.uri(), + originalModelServiceSettings.organizationId(), + originalModelServiceSettings.maxInputTokens(), + originalModelServiceSettings.rateLimitSettings() + ); + + return new OpenAiChatCompletionModel( + model.getInferenceEntityId(), + model.getTaskType(), + model.getConfigurations().getService(), + overriddenServiceSettings, + model.getTaskSettings(), + model.getSecretSettings() + ); + } + public OpenAiChatCompletionModel( String inferenceEntityId, TaskType taskType, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionRequestTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionRequestTaskSettings.java index 8029d8579baba..7ef7f85d71a6a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionRequestTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionRequestTaskSettings.java @@ -48,5 +48,4 @@ public static OpenAiChatCompletionRequestTaskSettings fromMap(Map TaskType.fromStringOrStatusException(null)); + assertThat(exception.getMessage(), Matchers.is("Task type must not be null")); + + exception = expectThrows(ElasticsearchStatusException.class, () -> TaskType.fromStringOrStatusException("blah")); + assertThat(exception.getMessage(), Matchers.is("Unknown task_type [blah]")); + + assertThat(TaskType.fromStringOrStatusException("any"), Matchers.is(TaskType.ANY)); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java index 5abb9000f4d04..9395ae222e9ba 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java @@ -19,6 +19,7 @@ import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.inference.TaskType; import org.elasticsearch.threadpool.ScalingExecutorBuilder; import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; import org.elasticsearch.xpack.inference.common.Truncator; @@ -160,9 +161,11 @@ public static Model getInvalidModel(String inferenceEntityId, String serviceName var mockConfigs = mock(ModelConfigurations.class); when(mockConfigs.getInferenceEntityId()).thenReturn(inferenceEntityId); when(mockConfigs.getService()).thenReturn(serviceName); + when(mockConfigs.getTaskType()).thenReturn(TaskType.TEXT_EMBEDDING); var mockModel = mock(Model.class); when(mockModel.getConfigurations()).thenReturn(mockConfigs); + when(mockModel.getTaskType()).thenReturn(TaskType.TEXT_EMBEDDING); return mockModel; } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java new file mode 100644 index 0000000000000..47f3a0e0b57aa --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java @@ -0,0 +1,364 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.inference.InferenceService; +import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.inference.action.BaseInferenceActionRequest; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; +import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.elasticsearch.xpack.inference.telemetry.InferenceStats; +import org.junit.Before; +import org.mockito.ArgumentCaptor; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Flow; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isA; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.assertArg; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public abstract class BaseTransportInferenceActionTestCase extends ESTestCase { + private ModelRegistry modelRegistry; + private StreamingTaskManager streamingTaskManager; + private BaseTransportInferenceAction action; + + protected static final String serviceId = "serviceId"; + protected static final TaskType taskType = TaskType.COMPLETION; + protected static final String inferenceId = "inferenceEntityId"; + protected InferenceServiceRegistry serviceRegistry; + protected InferenceStats inferenceStats; + + @Before + public void setUp() throws Exception { + super.setUp(); + TransportService transportService = mock(); + ActionFilters actionFilters = mock(); + modelRegistry = mock(); + serviceRegistry = mock(); + inferenceStats = new InferenceStats(mock(), mock()); + streamingTaskManager = mock(); + action = createAction(transportService, actionFilters, modelRegistry, serviceRegistry, inferenceStats, streamingTaskManager); + } + + protected abstract BaseTransportInferenceAction createAction( + TransportService transportService, + ActionFilters actionFilters, + ModelRegistry modelRegistry, + InferenceServiceRegistry serviceRegistry, + InferenceStats inferenceStats, + StreamingTaskManager streamingTaskManager + ); + + protected abstract Request createRequest(); + + public void testMetricsAfterModelRegistryError() { + var expectedException = new IllegalStateException("hello"); + var expectedError = expectedException.getClass().getSimpleName(); + + doAnswer(ans -> { + ActionListener listener = ans.getArgument(1); + listener.onFailure(expectedException); + return null; + }).when(modelRegistry).getModelWithSecrets(any(), any()); + + var listener = doExecute(taskType); + verify(listener).onFailure(same(expectedException)); + + verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { + assertThat(attributes.get("service"), nullValue()); + assertThat(attributes.get("task_type"), nullValue()); + assertThat(attributes.get("model_id"), nullValue()); + assertThat(attributes.get("status_code"), nullValue()); + assertThat(attributes.get("error.type"), is(expectedError)); + })); + } + + protected ActionListener doExecute(TaskType taskType) { + return doExecute(taskType, false); + } + + protected ActionListener doExecute(TaskType taskType, boolean stream) { + Request request = createRequest(); + when(request.getInferenceEntityId()).thenReturn(inferenceId); + when(request.getTaskType()).thenReturn(taskType); + when(request.isStreaming()).thenReturn(stream); + ActionListener listener = mock(); + action.doExecute(mock(), request, listener); + return listener; + } + + public void testMetricsAfterMissingService() { + mockModelRegistry(taskType); + + when(serviceRegistry.getService(any())).thenReturn(Optional.empty()); + + var listener = doExecute(taskType); + + verify(listener).onFailure(assertArg(e -> { + assertThat(e, isA(ElasticsearchStatusException.class)); + assertThat(e.getMessage(), is("Unknown service [" + serviceId + "] for model [" + inferenceId + "]. ")); + assertThat(((ElasticsearchStatusException) e).status(), is(RestStatus.BAD_REQUEST)); + })); + verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { + assertThat(attributes.get("service"), is(serviceId)); + assertThat(attributes.get("task_type"), is(taskType.toString())); + assertThat(attributes.get("model_id"), nullValue()); + assertThat(attributes.get("status_code"), is(RestStatus.BAD_REQUEST.getStatus())); + assertThat(attributes.get("error.type"), is(String.valueOf(RestStatus.BAD_REQUEST.getStatus()))); + })); + } + + protected void mockModelRegistry(TaskType expectedTaskType) { + var unparsedModel = new UnparsedModel(inferenceId, expectedTaskType, serviceId, Map.of(), Map.of()); + doAnswer(ans -> { + ActionListener listener = ans.getArgument(1); + listener.onResponse(unparsedModel); + return null; + }).when(modelRegistry).getModelWithSecrets(any(), any()); + } + + public void testMetricsAfterUnknownTaskType() { + var modelTaskType = TaskType.RERANK; + var requestTaskType = TaskType.SPARSE_EMBEDDING; + mockModelRegistry(modelTaskType); + when(serviceRegistry.getService(any())).thenReturn(Optional.of(mock())); + + var listener = doExecute(requestTaskType); + + verify(listener).onFailure(assertArg(e -> { + assertThat(e, isA(ElasticsearchStatusException.class)); + assertThat( + e.getMessage(), + is( + "Incompatible task_type, the requested type [" + + requestTaskType + + "] does not match the model type [" + + modelTaskType + + "]" + ) + ); + assertThat(((ElasticsearchStatusException) e).status(), is(RestStatus.BAD_REQUEST)); + })); + verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { + assertThat(attributes.get("service"), is(serviceId)); + assertThat(attributes.get("task_type"), is(modelTaskType.toString())); + assertThat(attributes.get("model_id"), nullValue()); + assertThat(attributes.get("status_code"), is(RestStatus.BAD_REQUEST.getStatus())); + assertThat(attributes.get("error.type"), is(String.valueOf(RestStatus.BAD_REQUEST.getStatus()))); + })); + } + + public void testMetricsAfterInferError() { + var expectedException = new IllegalStateException("hello"); + var expectedError = expectedException.getClass().getSimpleName(); + mockService(listener -> listener.onFailure(expectedException)); + + var listener = doExecute(taskType); + + verify(listener).onFailure(same(expectedException)); + verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { + assertThat(attributes.get("service"), is(serviceId)); + assertThat(attributes.get("task_type"), is(taskType.toString())); + assertThat(attributes.get("model_id"), nullValue()); + assertThat(attributes.get("status_code"), nullValue()); + assertThat(attributes.get("error.type"), is(expectedError)); + })); + } + + public void testMetricsAfterStreamUnsupported() { + var expectedStatus = RestStatus.METHOD_NOT_ALLOWED; + var expectedError = String.valueOf(expectedStatus.getStatus()); + mockService(l -> {}); + + var listener = doExecute(taskType, true); + + verify(listener).onFailure(assertArg(e -> { + assertThat(e, isA(ElasticsearchStatusException.class)); + var ese = (ElasticsearchStatusException) e; + assertThat(ese.getMessage(), is("Streaming is not allowed for service [" + serviceId + "].")); + assertThat(ese.status(), is(expectedStatus)); + })); + verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { + assertThat(attributes.get("service"), is(serviceId)); + assertThat(attributes.get("task_type"), is(taskType.toString())); + assertThat(attributes.get("model_id"), nullValue()); + assertThat(attributes.get("status_code"), is(expectedStatus.getStatus())); + assertThat(attributes.get("error.type"), is(expectedError)); + })); + } + + public void testMetricsAfterInferSuccess() { + mockService(listener -> listener.onResponse(mock())); + + var listener = doExecute(taskType); + + verify(listener).onResponse(any()); + verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { + assertThat(attributes.get("service"), is(serviceId)); + assertThat(attributes.get("task_type"), is(taskType.toString())); + assertThat(attributes.get("model_id"), nullValue()); + assertThat(attributes.get("status_code"), is(200)); + assertThat(attributes.get("error.type"), nullValue()); + })); + } + + public void testMetricsAfterStreamInferSuccess() { + mockStreamResponse(Flow.Subscriber::onComplete); + verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { + assertThat(attributes.get("service"), is(serviceId)); + assertThat(attributes.get("task_type"), is(taskType.toString())); + assertThat(attributes.get("model_id"), nullValue()); + assertThat(attributes.get("status_code"), is(200)); + assertThat(attributes.get("error.type"), nullValue()); + })); + } + + public void testMetricsAfterStreamInferFailure() { + var expectedException = new IllegalStateException("hello"); + var expectedError = expectedException.getClass().getSimpleName(); + mockStreamResponse(subscriber -> { + subscriber.subscribe(mock()); + subscriber.onError(expectedException); + }); + verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { + assertThat(attributes.get("service"), is(serviceId)); + assertThat(attributes.get("task_type"), is(taskType.toString())); + assertThat(attributes.get("model_id"), nullValue()); + assertThat(attributes.get("status_code"), nullValue()); + assertThat(attributes.get("error.type"), is(expectedError)); + })); + } + + public void testMetricsAfterStreamCancel() { + var response = mockStreamResponse(s -> s.onSubscribe(mock())); + response.subscribe(new Flow.Subscriber<>() { + @Override + public void onSubscribe(Flow.Subscription subscription) { + subscription.cancel(); + } + + @Override + public void onNext(ChunkedToXContent item) { + + } + + @Override + public void onError(Throwable throwable) { + + } + + @Override + public void onComplete() { + + } + }); + + verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { + assertThat(attributes.get("service"), is(serviceId)); + assertThat(attributes.get("task_type"), is(taskType.toString())); + assertThat(attributes.get("model_id"), nullValue()); + assertThat(attributes.get("status_code"), is(200)); + assertThat(attributes.get("error.type"), nullValue()); + })); + } + + protected Flow.Publisher mockStreamResponse(Consumer> action) { + mockService(true, Set.of(), listener -> { + Flow.Processor taskProcessor = mock(); + doAnswer(innerAns -> { + action.accept(innerAns.getArgument(0)); + return null; + }).when(taskProcessor).subscribe(any()); + when(streamingTaskManager.create(any(), any())).thenReturn(taskProcessor); + var inferenceServiceResults = mock(InferenceServiceResults.class); + when(inferenceServiceResults.publisher()).thenReturn(mock()); + listener.onResponse(inferenceServiceResults); + }); + + var listener = doExecute(taskType, true); + var captor = ArgumentCaptor.forClass(InferenceAction.Response.class); + verify(listener).onResponse(captor.capture()); + assertTrue(captor.getValue().isStreaming()); + assertNotNull(captor.getValue().publisher()); + return captor.getValue().publisher(); + } + + protected void mockService(Consumer> listenerAction) { + mockService(false, Set.of(), listenerAction); + } + + protected void mockService( + boolean stream, + Set supportedStreamingTasks, + Consumer> listenerAction + ) { + InferenceService service = mock(); + Model model = mockModel(); + when(service.parsePersistedConfigWithSecrets(any(), any(), any(), any())).thenReturn(model); + when(service.name()).thenReturn(serviceId); + + when(service.canStream(any())).thenReturn(stream); + when(service.supportedStreamingTasks()).thenReturn(supportedStreamingTasks); + doAnswer(ans -> { + listenerAction.accept(ans.getArgument(7)); + return null; + }).when(service).infer(any(), any(), any(), anyBoolean(), any(), any(), any(), any()); + doAnswer(ans -> { + listenerAction.accept(ans.getArgument(3)); + return null; + }).when(service).unifiedCompletionInfer(any(), any(), any(), any()); + mockModelAndServiceRegistry(service); + } + + protected Model mockModel() { + Model model = mock(); + ModelConfigurations modelConfigurations = mock(); + when(modelConfigurations.getService()).thenReturn(serviceId); + when(model.getConfigurations()).thenReturn(modelConfigurations); + when(model.getTaskType()).thenReturn(taskType); + when(model.getServiceSettings()).thenReturn(mock()); + return model; + } + + protected void mockModelAndServiceRegistry(InferenceService service) { + var unparsedModel = new UnparsedModel(inferenceId, taskType, serviceId, Map.of(), Map.of()); + doAnswer(ans -> { + ActionListener listener = ans.getArgument(1); + listener.onResponse(unparsedModel); + return null; + }).when(modelRegistry).getModelWithSecrets(any(), any()); + + when(serviceRegistry.getService(any())).thenReturn(Optional.of(service)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java index 0ed9cbf56b3fa..e54175cb27009 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java @@ -7,66 +7,28 @@ package org.elasticsearch.xpack.inference.action; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.common.xcontent.ChunkedToXContent; -import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.InferenceServiceRegistry; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.inference.Model; -import org.elasticsearch.inference.ModelConfigurations; -import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.UnparsedModel; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.telemetry.InferenceStats; -import org.junit.Before; -import org.mockito.ArgumentCaptor; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.Flow; -import java.util.function.Consumer; - -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.isA; -import static org.hamcrest.Matchers.nullValue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.assertArg; -import static org.mockito.ArgumentMatchers.same; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -public class TransportInferenceActionTests extends ESTestCase { - private static final String serviceId = "serviceId"; - private static final TaskType taskType = TaskType.COMPLETION; - private static final String inferenceId = "inferenceEntityId"; - private ModelRegistry modelRegistry; - private InferenceServiceRegistry serviceRegistry; - private InferenceStats inferenceStats; - private StreamingTaskManager streamingTaskManager; - private TransportInferenceAction action; +public class TransportInferenceActionTests extends BaseTransportInferenceActionTestCase { - @Before - public void setUp() throws Exception { - super.setUp(); - TransportService transportService = mock(); - ActionFilters actionFilters = mock(); - modelRegistry = mock(); - serviceRegistry = mock(); - inferenceStats = new InferenceStats(mock(), mock()); - streamingTaskManager = mock(); - action = new TransportInferenceAction( + @Override + protected BaseTransportInferenceAction createAction( + TransportService transportService, + ActionFilters actionFilters, + ModelRegistry modelRegistry, + InferenceServiceRegistry serviceRegistry, + InferenceStats inferenceStats, + StreamingTaskManager streamingTaskManager + ) { + return new TransportInferenceAction( transportService, actionFilters, modelRegistry, @@ -76,279 +38,8 @@ public void setUp() throws Exception { ); } - public void testMetricsAfterModelRegistryError() { - var expectedException = new IllegalStateException("hello"); - var expectedError = expectedException.getClass().getSimpleName(); - - doAnswer(ans -> { - ActionListener listener = ans.getArgument(1); - listener.onFailure(expectedException); - return null; - }).when(modelRegistry).getModelWithSecrets(any(), any()); - - var listener = doExecute(taskType); - verify(listener).onFailure(same(expectedException)); - - verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { - assertThat(attributes.get("service"), nullValue()); - assertThat(attributes.get("task_type"), nullValue()); - assertThat(attributes.get("model_id"), nullValue()); - assertThat(attributes.get("status_code"), nullValue()); - assertThat(attributes.get("error.type"), is(expectedError)); - })); - } - - private ActionListener doExecute(TaskType taskType) { - return doExecute(taskType, false); - } - - private ActionListener doExecute(TaskType taskType, boolean stream) { - InferenceAction.Request request = mock(); - when(request.getInferenceEntityId()).thenReturn(inferenceId); - when(request.getTaskType()).thenReturn(taskType); - when(request.isStreaming()).thenReturn(stream); - ActionListener listener = mock(); - action.doExecute(mock(), request, listener); - return listener; - } - - public void testMetricsAfterMissingService() { - mockModelRegistry(taskType); - - when(serviceRegistry.getService(any())).thenReturn(Optional.empty()); - - var listener = doExecute(taskType); - - verify(listener).onFailure(assertArg(e -> { - assertThat(e, isA(ElasticsearchStatusException.class)); - assertThat(e.getMessage(), is("Unknown service [" + serviceId + "] for model [" + inferenceId + "]. ")); - assertThat(((ElasticsearchStatusException) e).status(), is(RestStatus.BAD_REQUEST)); - })); - verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { - assertThat(attributes.get("service"), is(serviceId)); - assertThat(attributes.get("task_type"), is(taskType.toString())); - assertThat(attributes.get("model_id"), nullValue()); - assertThat(attributes.get("status_code"), is(RestStatus.BAD_REQUEST.getStatus())); - assertThat(attributes.get("error.type"), is(String.valueOf(RestStatus.BAD_REQUEST.getStatus()))); - })); - } - - private void mockModelRegistry(TaskType expectedTaskType) { - var unparsedModel = new UnparsedModel(inferenceId, expectedTaskType, serviceId, Map.of(), Map.of()); - doAnswer(ans -> { - ActionListener listener = ans.getArgument(1); - listener.onResponse(unparsedModel); - return null; - }).when(modelRegistry).getModelWithSecrets(any(), any()); - } - - public void testMetricsAfterUnknownTaskType() { - var modelTaskType = TaskType.RERANK; - var requestTaskType = TaskType.SPARSE_EMBEDDING; - mockModelRegistry(modelTaskType); - when(serviceRegistry.getService(any())).thenReturn(Optional.of(mock())); - - var listener = doExecute(requestTaskType); - - verify(listener).onFailure(assertArg(e -> { - assertThat(e, isA(ElasticsearchStatusException.class)); - assertThat( - e.getMessage(), - is( - "Incompatible task_type, the requested type [" - + requestTaskType - + "] does not match the model type [" - + modelTaskType - + "]" - ) - ); - assertThat(((ElasticsearchStatusException) e).status(), is(RestStatus.BAD_REQUEST)); - })); - verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { - assertThat(attributes.get("service"), is(serviceId)); - assertThat(attributes.get("task_type"), is(modelTaskType.toString())); - assertThat(attributes.get("model_id"), nullValue()); - assertThat(attributes.get("status_code"), is(RestStatus.BAD_REQUEST.getStatus())); - assertThat(attributes.get("error.type"), is(String.valueOf(RestStatus.BAD_REQUEST.getStatus()))); - })); - } - - public void testMetricsAfterInferError() { - var expectedException = new IllegalStateException("hello"); - var expectedError = expectedException.getClass().getSimpleName(); - mockService(listener -> listener.onFailure(expectedException)); - - var listener = doExecute(taskType); - - verify(listener).onFailure(same(expectedException)); - verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { - assertThat(attributes.get("service"), is(serviceId)); - assertThat(attributes.get("task_type"), is(taskType.toString())); - assertThat(attributes.get("model_id"), nullValue()); - assertThat(attributes.get("status_code"), nullValue()); - assertThat(attributes.get("error.type"), is(expectedError)); - })); - } - - public void testMetricsAfterStreamUnsupported() { - var expectedStatus = RestStatus.METHOD_NOT_ALLOWED; - var expectedError = String.valueOf(expectedStatus.getStatus()); - mockService(l -> {}); - - var listener = doExecute(taskType, true); - - verify(listener).onFailure(assertArg(e -> { - assertThat(e, isA(ElasticsearchStatusException.class)); - var ese = (ElasticsearchStatusException) e; - assertThat(ese.getMessage(), is("Streaming is not allowed for service [" + serviceId + "].")); - assertThat(ese.status(), is(expectedStatus)); - })); - verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { - assertThat(attributes.get("service"), is(serviceId)); - assertThat(attributes.get("task_type"), is(taskType.toString())); - assertThat(attributes.get("model_id"), nullValue()); - assertThat(attributes.get("status_code"), is(expectedStatus.getStatus())); - assertThat(attributes.get("error.type"), is(expectedError)); - })); - } - - public void testMetricsAfterInferSuccess() { - mockService(listener -> listener.onResponse(mock())); - - var listener = doExecute(taskType); - - verify(listener).onResponse(any()); - verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { - assertThat(attributes.get("service"), is(serviceId)); - assertThat(attributes.get("task_type"), is(taskType.toString())); - assertThat(attributes.get("model_id"), nullValue()); - assertThat(attributes.get("status_code"), is(200)); - assertThat(attributes.get("error.type"), nullValue()); - })); - } - - public void testMetricsAfterStreamInferSuccess() { - mockStreamResponse(Flow.Subscriber::onComplete); - verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { - assertThat(attributes.get("service"), is(serviceId)); - assertThat(attributes.get("task_type"), is(taskType.toString())); - assertThat(attributes.get("model_id"), nullValue()); - assertThat(attributes.get("status_code"), is(200)); - assertThat(attributes.get("error.type"), nullValue()); - })); - } - - public void testMetricsAfterStreamInferFailure() { - var expectedException = new IllegalStateException("hello"); - var expectedError = expectedException.getClass().getSimpleName(); - mockStreamResponse(subscriber -> { - subscriber.subscribe(mock()); - subscriber.onError(expectedException); - }); - verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { - assertThat(attributes.get("service"), is(serviceId)); - assertThat(attributes.get("task_type"), is(taskType.toString())); - assertThat(attributes.get("model_id"), nullValue()); - assertThat(attributes.get("status_code"), nullValue()); - assertThat(attributes.get("error.type"), is(expectedError)); - })); - } - - public void testMetricsAfterStreamCancel() { - var response = mockStreamResponse(s -> s.onSubscribe(mock())); - response.subscribe(new Flow.Subscriber<>() { - @Override - public void onSubscribe(Flow.Subscription subscription) { - subscription.cancel(); - } - - @Override - public void onNext(ChunkedToXContent item) { - - } - - @Override - public void onError(Throwable throwable) { - - } - - @Override - public void onComplete() { - - } - }); - - verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { - assertThat(attributes.get("service"), is(serviceId)); - assertThat(attributes.get("task_type"), is(taskType.toString())); - assertThat(attributes.get("model_id"), nullValue()); - assertThat(attributes.get("status_code"), is(200)); - assertThat(attributes.get("error.type"), nullValue()); - })); - } - - private Flow.Publisher mockStreamResponse(Consumer> action) { - mockService(true, Set.of(), listener -> { - Flow.Processor taskProcessor = mock(); - doAnswer(innerAns -> { - action.accept(innerAns.getArgument(0)); - return null; - }).when(taskProcessor).subscribe(any()); - when(streamingTaskManager.create(any(), any())).thenReturn(taskProcessor); - var inferenceServiceResults = mock(InferenceServiceResults.class); - when(inferenceServiceResults.publisher()).thenReturn(mock()); - listener.onResponse(inferenceServiceResults); - }); - - var listener = doExecute(taskType, true); - var captor = ArgumentCaptor.forClass(InferenceAction.Response.class); - verify(listener).onResponse(captor.capture()); - assertTrue(captor.getValue().isStreaming()); - assertNotNull(captor.getValue().publisher()); - return captor.getValue().publisher(); - } - - private void mockService(Consumer> listenerAction) { - mockService(false, Set.of(), listenerAction); - } - - private void mockService( - boolean stream, - Set supportedStreamingTasks, - Consumer> listenerAction - ) { - InferenceService service = mock(); - Model model = mockModel(); - when(service.parsePersistedConfigWithSecrets(any(), any(), any(), any())).thenReturn(model); - when(service.name()).thenReturn(serviceId); - - when(service.canStream(any())).thenReturn(stream); - when(service.supportedStreamingTasks()).thenReturn(supportedStreamingTasks); - doAnswer(ans -> { - listenerAction.accept(ans.getArgument(7)); - return null; - }).when(service).infer(any(), any(), any(), anyBoolean(), any(), any(), any(), any()); - mockModelAndServiceRegistry(service); - } - - private Model mockModel() { - Model model = mock(); - ModelConfigurations modelConfigurations = mock(); - when(modelConfigurations.getService()).thenReturn(serviceId); - when(model.getConfigurations()).thenReturn(modelConfigurations); - when(model.getTaskType()).thenReturn(taskType); - when(model.getServiceSettings()).thenReturn(mock()); - return model; - } - - private void mockModelAndServiceRegistry(InferenceService service) { - var unparsedModel = new UnparsedModel(inferenceId, taskType, serviceId, Map.of(), Map.of()); - doAnswer(ans -> { - ActionListener listener = ans.getArgument(1); - listener.onResponse(unparsedModel); - return null; - }).when(modelRegistry).getModelWithSecrets(any(), any()); - - when(serviceRegistry.getService(any())).thenReturn(Optional.of(service)); + @Override + protected InferenceAction.Request createRequest() { + return mock(); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java new file mode 100644 index 0000000000000..4c943599ce523 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; +import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; +import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.elasticsearch.xpack.inference.telemetry.InferenceStats; + +import java.util.Optional; + +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isA; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.assertArg; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TransportUnifiedCompletionActionTests extends BaseTransportInferenceActionTestCase { + + @Override + protected BaseTransportInferenceAction createAction( + TransportService transportService, + ActionFilters actionFilters, + ModelRegistry modelRegistry, + InferenceServiceRegistry serviceRegistry, + InferenceStats inferenceStats, + StreamingTaskManager streamingTaskManager + ) { + return new TransportUnifiedCompletionInferenceAction( + transportService, + actionFilters, + modelRegistry, + serviceRegistry, + inferenceStats, + streamingTaskManager + ); + } + + @Override + protected UnifiedCompletionAction.Request createRequest() { + return mock(); + } + + public void testThrows_IncompatibleTaskTypeException_WhenUsingATextEmbeddingInferenceEndpoint() { + var modelTaskType = TaskType.TEXT_EMBEDDING; + var requestTaskType = TaskType.TEXT_EMBEDDING; + mockModelRegistry(modelTaskType); + when(serviceRegistry.getService(any())).thenReturn(Optional.of(mock())); + + var listener = doExecute(requestTaskType); + + verify(listener).onFailure(assertArg(e -> { + assertThat(e, isA(ElasticsearchStatusException.class)); + assertThat( + e.getMessage(), + is("Incompatible task_type for unified API, the requested type [" + requestTaskType + "] must be one of [completion]") + ); + assertThat(((ElasticsearchStatusException) e).status(), is(RestStatus.BAD_REQUEST)); + })); + verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { + assertThat(attributes.get("service"), is(serviceId)); + assertThat(attributes.get("task_type"), is(modelTaskType.toString())); + assertThat(attributes.get("model_id"), nullValue()); + assertThat(attributes.get("status_code"), is(RestStatus.BAD_REQUEST.getStatus())); + assertThat(attributes.get("error.type"), is(String.valueOf(RestStatus.BAD_REQUEST.getStatus()))); + })); + } + + public void testThrows_IncompatibleTaskTypeException_WhenUsingRequestIsAny_ModelIsTextEmbedding() { + var modelTaskType = TaskType.ANY; + var requestTaskType = TaskType.TEXT_EMBEDDING; + mockModelRegistry(modelTaskType); + when(serviceRegistry.getService(any())).thenReturn(Optional.of(mock())); + + var listener = doExecute(requestTaskType); + + verify(listener).onFailure(assertArg(e -> { + assertThat(e, isA(ElasticsearchStatusException.class)); + assertThat( + e.getMessage(), + is("Incompatible task_type for unified API, the requested type [" + requestTaskType + "] must be one of [completion]") + ); + assertThat(((ElasticsearchStatusException) e).status(), is(RestStatus.BAD_REQUEST)); + })); + verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { + assertThat(attributes.get("service"), is(serviceId)); + assertThat(attributes.get("task_type"), is(modelTaskType.toString())); + assertThat(attributes.get("model_id"), nullValue()); + assertThat(attributes.get("status_code"), is(RestStatus.BAD_REQUEST.getStatus())); + assertThat(attributes.get("error.type"), is(String.valueOf(RestStatus.BAD_REQUEST.getStatus()))); + })); + } + + public void testMetricsAfterUnifiedInferSuccess_WithRequestTaskTypeAny() { + mockModelRegistry(TaskType.COMPLETION); + mockService(listener -> listener.onResponse(mock())); + + var listener = doExecute(TaskType.ANY); + + verify(listener).onResponse(any()); + verify(inferenceStats.inferenceDuration()).record(anyLong(), assertArg(attributes -> { + assertThat(attributes.get("service"), is(serviceId)); + assertThat(attributes.get("task_type"), is(taskType.toString())); + assertThat(attributes.get("model_id"), nullValue()); + assertThat(attributes.get("status_code"), is(200)); + assertThat(attributes.get("error.type"), nullValue()); + })); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableActionTests.java index d4ab9b1f1e19a..9e7c58b0ca79e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/SingleInputSenderExecutableActionTests.java @@ -61,25 +61,11 @@ public void testOneInputIsValid() { assertTrue("Test failed to call listener.", testRan.get()); } - public void testInvalidInputType() { - var badInput = mock(InferenceInputs.class); - var actualException = new AtomicReference(); - - executableAction.execute( - badInput, - mock(TimeValue.class), - ActionListener.wrap(shouldNotSucceed -> fail("Test failed."), actualException::set) - ); - - assertThat(actualException.get(), notNullValue()); - assertThat(actualException.get().getMessage(), is("Invalid inference input type")); - assertThat(actualException.get(), instanceOf(ElasticsearchStatusException.class)); - assertThat(((ElasticsearchStatusException) actualException.get()).status(), is(RestStatus.INTERNAL_SERVER_ERROR)); - } - public void testMoreThanOneInput() { var badInput = mock(DocumentsOnlyInput.class); - when(badInput.getInputs()).thenReturn(List.of("one", "two")); + var input = List.of("one", "two"); + when(badInput.getInputs()).thenReturn(input); + when(badInput.inputSize()).thenReturn(input.size()); var actualException = new AtomicReference(); executableAction.execute( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreatorTests.java index 87d3a82b4aae6..e7543aa6ba9e5 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreatorTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.amazonbedrock.AmazonBedrockMockRequestSender; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.services.ServiceComponentsTests; import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProvider; @@ -130,7 +131,7 @@ public void testCompletionRequestAction() throws IOException { ); var action = creator.create(model, Map.of()); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); assertThat(result.asMap(), is(buildExpectationCompletion(List.of("test input string")))); @@ -163,7 +164,7 @@ public void testChatCompletionRequestAction_HandlesException() throws IOExceptio ); var action = creator.create(model, Map.of()); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); assertThat(sender.sendCount(), is(1)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreatorTests.java index a3114300c5ddc..f0de37ceaaf98 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicActionCreatorTests.java @@ -20,7 +20,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.request.anthropic.AnthropicRequestUtils; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; @@ -49,6 +49,7 @@ import static org.mockito.Mockito.mock; public class AnthropicActionCreatorTests extends ESTestCase { + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); private final MockWebServer webServer = new MockWebServer(); private ThreadPool threadPool; @@ -103,7 +104,7 @@ public void testCreate_ChatCompletionModel() throws IOException { var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -168,7 +169,7 @@ public void testCreate_ChatCompletionModel_FailsFromInvalidResponseFormat() thro var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionActionTests.java index fca2e316af17f..2065a726b7589 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/anthropic/AnthropicChatCompletionActionTests.java @@ -27,7 +27,7 @@ import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.AnthropicCompletionRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.http.sender.Sender; @@ -113,7 +113,7 @@ public void testExecute_ReturnsSuccessfulResponse() throws IOException { var action = createAction(getUrl(webServer), "secret", "model", 1, sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -149,7 +149,7 @@ public void testExecute_ThrowsElasticsearchException() { var action = createAction(getUrl(webServer), "secret", "model", 1, sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -170,7 +170,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var action = createAction(getUrl(webServer), "secret", "model", 1, sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -187,7 +187,7 @@ public void testExecute_ThrowsException() { var action = createAction(getUrl(webServer), "secret", "model", 1, sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -229,7 +229,7 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc var action = createAction(getUrl(webServer), "secret", "model", 1, sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc", "def")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc", "def")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionAndCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionAndCreatorTests.java index 8792234102a94..210fab457de10 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionAndCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionAndCreatorTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.common.TruncatorTests; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -160,7 +161,7 @@ public void testChatCompletionRequestAction() throws IOException { var action = creator.create(model, Map.of()); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java index 45a2fb0954c79..7e1e3e55caed8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.request.azureopenai.AzureOpenAiUtils; @@ -475,7 +476,7 @@ public void testInfer_AzureOpenAiCompletion_WithOverriddenUser() throws IOExcept var action = actionCreator.create(model, taskSettingsWithUserOverride); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of(completionInput)), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of(completionInput)), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -531,7 +532,7 @@ public void testInfer_AzureOpenAiCompletionModel_WithoutUser() throws IOExceptio var action = actionCreator.create(model, requestTaskSettingsWithoutUser); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of(completionInput)), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of(completionInput)), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -589,7 +590,7 @@ public void testInfer_AzureOpenAiCompletionModel_FailsFromInvalidResponseFormat( var action = actionCreator.create(model, requestTaskSettingsWithoutUser); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of(completionInput)), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of(completionInput)), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java index 4c7683c882816..dca12dfda9c98 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.AzureOpenAiCompletionRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.external.request.azureopenai.AzureOpenAiUtils; @@ -111,7 +111,7 @@ public void testExecute_ReturnsSuccessfulResponse() throws IOException { var action = createAction("resource", "deployment", "apiversion", user, apiKey, sender, "id"); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of(completionInput)), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of(completionInput)), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -142,7 +142,7 @@ public void testExecute_ThrowsElasticsearchException() { var action = createAction("resource", "deployment", "apiVersion", "user", "apikey", sender, "id"); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -163,7 +163,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var action = createAction("resource", "deployment", "apiVersion", "user", "apikey", sender, "id"); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -177,7 +177,7 @@ public void testExecute_ThrowsException() { var action = createAction("resource", "deployment", "apiVersion", "user", "apikey", sender, "id"); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreatorTests.java index 9ec34e7d8e5c5..3a512de25a39c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreatorTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; @@ -197,7 +198,7 @@ public void testCreate_CohereCompletionModel_WithModelSpecified() throws IOExcep var action = actionCreator.create(model, Map.of()); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -257,7 +258,7 @@ public void testCreate_CohereCompletionModel_WithoutModelSpecified() throws IOEx var action = actionCreator.create(model, Map.of()); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java index ba839e0d7c5e9..c5871adb34864 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java @@ -26,8 +26,8 @@ import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.CohereCompletionRequestManager; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.external.request.cohere.CohereUtils; @@ -120,7 +120,7 @@ public void testExecute_ReturnsSuccessfulResponse_WithModelSpecified() throws IO var action = createAction(getUrl(webServer), "secret", "model", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -181,7 +181,7 @@ public void testExecute_ReturnsSuccessfulResponse_WithoutModelSpecified() throws var action = createAction(getUrl(webServer), "secret", null, sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -214,7 +214,7 @@ public void testExecute_ThrowsElasticsearchException() { var action = createAction(getUrl(webServer), "secret", "model", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -235,7 +235,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var action = createAction(getUrl(webServer), "secret", "model", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -256,7 +256,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var action = createAction(null, "secret", "model", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -270,7 +270,7 @@ public void testExecute_ThrowsException() { var action = createAction(getUrl(webServer), "secret", "model", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -284,7 +284,7 @@ public void testExecute_ThrowsExceptionWithNullUrl() { var action = createAction(null, "secret", "model", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -334,7 +334,7 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc var action = createAction(getUrl(webServer), "secret", "model", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc", "def")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc", "def")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java index 72b5ffa45a0dd..ff17bbf66e02a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java @@ -25,7 +25,7 @@ import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioCompletionRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.http.sender.Sender; @@ -128,7 +128,7 @@ public void testExecute_ReturnsSuccessfulResponse() throws IOException { var action = createAction(getUrl(webServer), "secret", "model", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("input")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("input")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -159,7 +159,7 @@ public void testExecute_ThrowsElasticsearchException() { var action = createAction(getUrl(webServer), "secret", "model", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -180,7 +180,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var action = createAction(getUrl(webServer), "secret", "model", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -197,7 +197,7 @@ public void testExecute_ThrowsException() { var action = createAction(getUrl(webServer), "secret", "model", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -260,7 +260,7 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc var action = createAction(getUrl(webServer), "secret", "model", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc", "def")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc", "def")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java index b6d7eb673b7f0..fe076eb721ea2 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; @@ -330,7 +331,7 @@ public void testCreate_OpenAiChatCompletionModel() throws IOException { var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -345,11 +346,12 @@ public void testCreate_OpenAiChatCompletionModel() throws IOException { assertThat(request.getHeader(ORGANIZATION_HEADER), equalTo("org")); var requestMap = entityAsMap(webServer.requests().get(0).getBody()); - assertThat(requestMap.size(), is(4)); + assertThat(requestMap.size(), is(5)); assertThat(requestMap.get("messages"), is(List.of(Map.of("role", "user", "content", "abc")))); assertThat(requestMap.get("model"), is("model")); assertThat(requestMap.get("user"), is("overridden_user")); assertThat(requestMap.get("n"), is(1)); + assertThat(requestMap.get("stream"), is(false)); } } @@ -393,7 +395,7 @@ public void testCreate_OpenAiChatCompletionModel_WithoutUser() throws IOExceptio var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -408,10 +410,11 @@ public void testCreate_OpenAiChatCompletionModel_WithoutUser() throws IOExceptio assertThat(request.getHeader(ORGANIZATION_HEADER), equalTo("org")); var requestMap = entityAsMap(webServer.requests().get(0).getBody()); - assertThat(requestMap.size(), is(3)); + assertThat(requestMap.size(), is(4)); assertThat(requestMap.get("messages"), is(List.of(Map.of("role", "user", "content", "abc")))); assertThat(requestMap.get("model"), is("model")); assertThat(requestMap.get("n"), is(1)); + assertThat(requestMap.get("stream"), is(false)); } } @@ -455,7 +458,7 @@ public void testCreate_OpenAiChatCompletionModel_WithoutOrganization() throws IO var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -470,11 +473,12 @@ public void testCreate_OpenAiChatCompletionModel_WithoutOrganization() throws IO assertNull(request.getHeader(ORGANIZATION_HEADER)); var requestMap = entityAsMap(webServer.requests().get(0).getBody()); - assertThat(requestMap.size(), is(4)); + assertThat(requestMap.size(), is(5)); assertThat(requestMap.get("messages"), is(List.of(Map.of("role", "user", "content", "abc")))); assertThat(requestMap.get("model"), is("model")); assertThat(requestMap.get("user"), is("overridden_user")); assertThat(requestMap.get("n"), is(1)); + assertThat(requestMap.get("stream"), is(false)); } } @@ -523,7 +527,7 @@ public void testCreate_OpenAiChatCompletionModel_FailsFromInvalidResponseFormat( var action = actionCreator.create(model, overriddenTaskSettings); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( @@ -542,11 +546,12 @@ public void testCreate_OpenAiChatCompletionModel_FailsFromInvalidResponseFormat( assertNull(webServer.requests().get(0).getHeader(ORGANIZATION_HEADER)); var requestMap = entityAsMap(webServer.requests().get(0).getBody()); - assertThat(requestMap.size(), is(4)); + assertThat(requestMap.size(), is(5)); assertThat(requestMap.get("messages"), is(List.of(Map.of("role", "user", "content", "abc")))); assertThat(requestMap.get("model"), is("model")); assertThat(requestMap.get("user"), is("overridden_user")); assertThat(requestMap.get("n"), is(1)); + assertThat(requestMap.get("stream"), is(false)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java index d84b2b5bb324a..ba74d2ab42c21 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java @@ -27,7 +27,7 @@ import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.SingleInputSenderExecutableAction; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.http.sender.OpenAiCompletionRequestManager; @@ -119,7 +119,7 @@ public void testExecute_ReturnsSuccessfulResponse() throws IOException { var action = createAction(getUrl(webServer), "org", "secret", "model", "user", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var result = listener.actionGet(TIMEOUT); @@ -134,11 +134,12 @@ public void testExecute_ReturnsSuccessfulResponse() throws IOException { assertThat(request.getHeader(ORGANIZATION_HEADER), equalTo("org")); var requestMap = entityAsMap(request.getBody()); - assertThat(requestMap.size(), is(4)); + assertThat(requestMap.size(), is(5)); assertThat(requestMap.get("messages"), is(List.of(Map.of("role", "user", "content", "abc")))); assertThat(requestMap.get("model"), is("model")); assertThat(requestMap.get("user"), is("user")); assertThat(requestMap.get("n"), is(1)); + assertThat(requestMap.get("stream"), is(false)); } } @@ -159,7 +160,7 @@ public void testExecute_ThrowsElasticsearchException() { var action = createAction(getUrl(webServer), "org", "secret", "model", "user", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -180,7 +181,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var action = createAction(getUrl(webServer), "org", "secret", "model", "user", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -201,7 +202,7 @@ public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled var action = createAction(null, "org", "secret", "model", "user", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -215,7 +216,7 @@ public void testExecute_ThrowsException() { var action = createAction(getUrl(webServer), "org", "secret", "model", "user", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -229,7 +230,7 @@ public void testExecute_ThrowsExceptionWithNullUrl() { var action = createAction(null, "org", "secret", "model", "user", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); @@ -273,7 +274,7 @@ public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOExc var action = createAction(getUrl(webServer), "org", "secret", "model", "user", sender); PlainActionFuture listener = new PlainActionFuture<>(); - action.execute(new DocumentsOnlyInput(List.of("abc", "def")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + action.execute(new ChatCompletionInput(List.of("abc", "def")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockMockRequestSender.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockMockRequestSender.java index e68beaf4c1eb5..929aefeeef6b9 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockMockRequestSender.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockMockRequestSender.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.external.http.sender.RequestManager; @@ -67,8 +68,15 @@ public void send( ActionListener listener ) { sendCounter++; - var docsInput = (DocumentsOnlyInput) inferenceInputs; - inputs.add(docsInput.getInputs()); + if (inferenceInputs instanceof DocumentsOnlyInput docsInput) { + inputs.add(docsInput.getInputs()); + } else if (inferenceInputs instanceof ChatCompletionInput chatCompletionInput) { + inputs.add(chatCompletionInput.getInputs()); + } else { + throw new IllegalArgumentException( + "Invalid inference inputs received in mock sender: " + inferenceInputs.getClass().getSimpleName() + ); + } if (results.isEmpty()) { listener.onFailure(new ElasticsearchException("No results found")); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockRequestSenderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockRequestSenderTests.java index 7fa8a09d5bf12..a8f37aedcece3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockRequestSenderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockRequestSenderTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.inference.external.http.sender.AmazonBedrockChatCompletionRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.AmazonBedrockEmbeddingsRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; @@ -107,7 +108,7 @@ public void testCreateSender_SendsCompletionRequestAndReceivesResponse() throws PlainActionFuture listener = new PlainActionFuture<>(); var requestManager = new AmazonBedrockChatCompletionRequestManager(model, threadPool, new TimeValue(30, TimeUnit.SECONDS)); - sender.send(requestManager, new DocumentsOnlyInput(List.of("abc")), null, listener); + sender.send(requestManager, new ChatCompletionInput(List.of("abc")), null, listener); var result = listener.actionGet(TIMEOUT); assertThat(result.asMap(), is(buildExpectationCompletion(List.of("test response text")))); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceInputsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceInputsTests.java new file mode 100644 index 0000000000000..f0da67a982374 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceInputsTests.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.inference.UnifiedCompletionRequest; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +import java.util.List; + +public class InferenceInputsTests extends ESTestCase { + public void testCastToSucceeds() { + InferenceInputs inputs = new DocumentsOnlyInput(List.of(), false); + assertThat(inputs.castTo(DocumentsOnlyInput.class), Matchers.instanceOf(DocumentsOnlyInput.class)); + + var emptyRequest = new UnifiedCompletionRequest(List.of(), null, null, null, null, null, null, null); + assertThat(new UnifiedChatInput(emptyRequest, false).castTo(UnifiedChatInput.class), Matchers.instanceOf(UnifiedChatInput.class)); + assertThat( + new QueryAndDocsInputs("hello", List.of(), false).castTo(QueryAndDocsInputs.class), + Matchers.instanceOf(QueryAndDocsInputs.class) + ); + } + + public void testCastToFails() { + InferenceInputs inputs = new DocumentsOnlyInput(List.of(), false); + var exception = expectThrows(IllegalArgumentException.class, () -> inputs.castTo(QueryAndDocsInputs.class)); + assertThat( + exception.getMessage(), + Matchers.containsString( + Strings.format("Unable to convert inference inputs type: [%s] to [%s]", DocumentsOnlyInput.class, QueryAndDocsInputs.class) + ) + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInputTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInputTests.java new file mode 100644 index 0000000000000..42e1b18168aec --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/UnifiedChatInputTests.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.inference.UnifiedCompletionRequest; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +import java.util.List; + +public class UnifiedChatInputTests extends ESTestCase { + + public void testConvertsStringInputToMessages() { + var a = new UnifiedChatInput(List.of("hello", "awesome"), "a role", true); + + assertThat(a.inputSize(), Matchers.is(2)); + assertThat( + a.getRequest(), + Matchers.is( + UnifiedCompletionRequest.of( + List.of( + new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString("hello"), + "a role", + null, + null, + null + ), + new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString("awesome"), + "a role", + null, + null, + null + ) + ) + ) + ) + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessorTests.java new file mode 100644 index 0000000000000..0f127998f9c54 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiUnifiedStreamingProcessorTests.java @@ -0,0 +1,383 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.openai; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.results.StreamingUnifiedChatCompletionResults; + +import java.io.IOException; +import java.util.List; + +public class OpenAiUnifiedStreamingProcessorTests extends ESTestCase { + + public void testJsonLiteral() { + String json = """ + { + "id": "example_id", + "choices": [ + { + "delta": { + "content": "example_content", + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "index": 1, + "id": "tool_call_id", + "function": { + "arguments": "example_arguments", + "name": "example_function_name" + }, + "type": "function" + } + ] + }, + "finish_reason": "stop", + "index": 0 + } + ], + "model": "example_model", + "object": "chat.completion.chunk", + "usage": { + "completion_tokens": 50, + "prompt_tokens": 20, + "total_tokens": 70 + } + } + """; + // Parse the JSON + XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler( + LoggingDeprecationHandler.INSTANCE + ); + try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, json)) { + StreamingUnifiedChatCompletionResults.ChatCompletionChunk chunk = OpenAiUnifiedStreamingProcessor.ChatCompletionChunkParser + .parse(parser); + + // Assertions to verify the parsed object + assertEquals("example_id", chunk.getId()); + assertEquals("example_model", chunk.getModel()); + assertEquals("chat.completion.chunk", chunk.getObject()); + assertNotNull(chunk.getUsage()); + assertEquals(50, chunk.getUsage().completionTokens()); + assertEquals(20, chunk.getUsage().promptTokens()); + assertEquals(70, chunk.getUsage().totalTokens()); + + List choices = chunk.getChoices(); + assertEquals(1, choices.size()); + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice choice = choices.get(0); + assertEquals("example_content", choice.delta().getContent()); + assertNull(choice.delta().getRefusal()); + assertEquals("assistant", choice.delta().getRole()); + assertEquals("stop", choice.finishReason()); + assertEquals(0, choice.index()); + + List toolCalls = choice.delta().getToolCalls(); + assertEquals(1, toolCalls.size()); + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall toolCall = toolCalls.get(0); + assertEquals(1, toolCall.getIndex()); + assertEquals("tool_call_id", toolCall.getId()); + assertEquals("example_function_name", toolCall.getFunction().getName()); + assertEquals("example_arguments", toolCall.getFunction().getArguments()); + assertEquals("function", toolCall.getType()); + } catch (IOException e) { + fail(); + } + } + + public void testJsonLiteralCornerCases() { + String json = """ + { + "id": "example_id", + "choices": [ + { + "delta": { + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [] + }, + "finish_reason": null, + "index": 0 + }, + { + "delta": { + "content": "example_content", + "refusal": "example_refusal", + "role": "user", + "tool_calls": [ + { + "index": 1, + "function": { + "name": "example_function_name" + }, + "type": "function" + } + ] + }, + "finish_reason": "stop", + "index": 1 + } + ], + "model": "example_model", + "object": "chat.completion.chunk", + "usage": null + } + """; + // Parse the JSON + XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler( + LoggingDeprecationHandler.INSTANCE + ); + try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, json)) { + StreamingUnifiedChatCompletionResults.ChatCompletionChunk chunk = OpenAiUnifiedStreamingProcessor.ChatCompletionChunkParser + .parse(parser); + + // Assertions to verify the parsed object + assertEquals("example_id", chunk.getId()); + assertEquals("example_model", chunk.getModel()); + assertEquals("chat.completion.chunk", chunk.getObject()); + assertNull(chunk.getUsage()); + + List choices = chunk.getChoices(); + assertEquals(2, choices.size()); + + // First choice assertions + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice firstChoice = choices.get(0); + assertNull(firstChoice.delta().getContent()); + assertNull(firstChoice.delta().getRefusal()); + assertEquals("assistant", firstChoice.delta().getRole()); + assertTrue(firstChoice.delta().getToolCalls().isEmpty()); + assertNull(firstChoice.finishReason()); + assertEquals(0, firstChoice.index()); + + // Second choice assertions + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice secondChoice = choices.get(1); + assertEquals("example_content", secondChoice.delta().getContent()); + assertEquals("example_refusal", secondChoice.delta().getRefusal()); + assertEquals("user", secondChoice.delta().getRole()); + assertEquals("stop", secondChoice.finishReason()); + assertEquals(1, secondChoice.index()); + + List toolCalls = secondChoice.delta() + .getToolCalls(); + assertEquals(1, toolCalls.size()); + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall toolCall = toolCalls.get(0); + assertEquals(1, toolCall.getIndex()); + assertNull(toolCall.getId()); + assertEquals("example_function_name", toolCall.getFunction().getName()); + assertNull(toolCall.getFunction().getArguments()); + assertEquals("function", toolCall.getType()); + } catch (IOException e) { + fail(); + } + } + + public void testOpenAiUnifiedStreamingProcessorParsing() throws IOException { + // Generate random values for the JSON fields + int toolCallIndex = randomIntBetween(0, 10); + String toolCallId = randomAlphaOfLength(5); + String toolCallFunctionName = randomAlphaOfLength(8); + String toolCallFunctionArguments = randomAlphaOfLength(10); + String toolCallType = "function"; + String toolCallJson = createToolCallJson(toolCallIndex, toolCallId, toolCallFunctionName, toolCallFunctionArguments, toolCallType); + + String choiceContent = randomAlphaOfLength(10); + String choiceRole = randomFrom("system", "user", "assistant", "tool"); + String choiceFinishReason = randomFrom("stop", "length", "tool_calls", "content_filter", "function_call", null); + int choiceIndex = randomIntBetween(0, 10); + String choiceJson = createChoiceJson(choiceContent, null, choiceRole, toolCallJson, choiceFinishReason, choiceIndex); + + int usageCompletionTokens = randomIntBetween(1, 100); + int usagePromptTokens = randomIntBetween(1, 100); + int usageTotalTokens = randomIntBetween(1, 200); + String usageJson = createUsageJson(usageCompletionTokens, usagePromptTokens, usageTotalTokens); + + String chatCompletionChunkId = randomAlphaOfLength(10); + String chatCompletionChunkModel = randomAlphaOfLength(5); + String chatCompletionChunkJson = createChatCompletionChunkJson( + chatCompletionChunkId, + choiceJson, + chatCompletionChunkModel, + "chat.completion.chunk", + usageJson + ); + + // Parse the JSON + XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler( + LoggingDeprecationHandler.INSTANCE + ); + try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, chatCompletionChunkJson)) { + StreamingUnifiedChatCompletionResults.ChatCompletionChunk chunk = OpenAiUnifiedStreamingProcessor.ChatCompletionChunkParser + .parse(parser); + + // Assertions to verify the parsed object + assertEquals(chatCompletionChunkId, chunk.getId()); + assertEquals(chatCompletionChunkModel, chunk.getModel()); + assertEquals("chat.completion.chunk", chunk.getObject()); + assertNotNull(chunk.getUsage()); + assertEquals(usageCompletionTokens, chunk.getUsage().completionTokens()); + assertEquals(usagePromptTokens, chunk.getUsage().promptTokens()); + assertEquals(usageTotalTokens, chunk.getUsage().totalTokens()); + + List choices = chunk.getChoices(); + assertEquals(1, choices.size()); + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice choice = choices.get(0); + assertEquals(choiceContent, choice.delta().getContent()); + assertNull(choice.delta().getRefusal()); + assertEquals(choiceRole, choice.delta().getRole()); + assertEquals(choiceFinishReason, choice.finishReason()); + assertEquals(choiceIndex, choice.index()); + + List toolCalls = choice.delta().getToolCalls(); + assertEquals(1, toolCalls.size()); + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta.ToolCall toolCall = toolCalls.get(0); + assertEquals(toolCallIndex, toolCall.getIndex()); + assertEquals(toolCallId, toolCall.getId()); + assertEquals(toolCallFunctionName, toolCall.getFunction().getName()); + assertEquals(toolCallFunctionArguments, toolCall.getFunction().getArguments()); + assertEquals(toolCallType, toolCall.getType()); + } + } + + public void testOpenAiUnifiedStreamingProcessorParsingWithNullFields() throws IOException { + // JSON with null fields + int choiceIndex = randomIntBetween(0, 10); + String choiceJson = createChoiceJson(null, null, null, "", null, choiceIndex); + + String chatCompletionChunkId = randomAlphaOfLength(10); + String chatCompletionChunkModel = randomAlphaOfLength(5); + String chatCompletionChunkJson = createChatCompletionChunkJson( + chatCompletionChunkId, + choiceJson, + chatCompletionChunkModel, + "chat.completion.chunk", + null + ); + + // Parse the JSON + XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler( + LoggingDeprecationHandler.INSTANCE + ); + try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, chatCompletionChunkJson)) { + StreamingUnifiedChatCompletionResults.ChatCompletionChunk chunk = OpenAiUnifiedStreamingProcessor.ChatCompletionChunkParser + .parse(parser); + + // Assertions to verify the parsed object + assertEquals(chatCompletionChunkId, chunk.getId()); + assertEquals(chatCompletionChunkModel, chunk.getModel()); + assertEquals("chat.completion.chunk", chunk.getObject()); + assertNull(chunk.getUsage()); + + List choices = chunk.getChoices(); + assertEquals(1, choices.size()); + StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice choice = choices.get(0); + assertNull(choice.delta().getContent()); + assertNull(choice.delta().getRefusal()); + assertNull(choice.delta().getRole()); + assertNull(choice.finishReason()); + assertEquals(choiceIndex, choice.index()); + assertTrue(choice.delta().getToolCalls().isEmpty()); + } + } + + private String createToolCallJson(int index, String id, String functionName, String functionArguments, String type) { + return Strings.format(""" + { + "index": %d, + "id": "%s", + "function": { + "name": "%s", + "arguments": "%s" + }, + "type": "%s" + } + """, index, id, functionName, functionArguments, type); + } + + private String createChoiceJson(String content, String refusal, String role, String toolCallsJson, String finishReason, int index) { + if (role == null) { + return Strings.format( + """ + { + "delta": { + "content": %s, + "refusal": %s, + "tool_calls": [%s] + }, + "finish_reason": %s, + "index": %d + } + """, + content != null ? "\"" + content + "\"" : "null", + refusal != null ? "\"" + refusal + "\"" : "null", + toolCallsJson, + finishReason != null ? "\"" + finishReason + "\"" : "null", + index + ); + } else { + return Strings.format( + """ + { + "delta": { + "content": %s, + "refusal": %s, + "role": %s, + "tool_calls": [%s] + }, + "finish_reason": %s, + "index": %d + } + """, + content != null ? "\"" + content + "\"" : "null", + refusal != null ? "\"" + refusal + "\"" : "null", + role != null ? "\"" + role + "\"" : "null", + toolCallsJson, + finishReason != null ? "\"" + finishReason + "\"" : "null", + index + ); + } + } + + private String createChatCompletionChunkJson(String id, String choicesJson, String model, String object, String usageJson) { + if (usageJson != null) { + return Strings.format(""" + { + "id": "%s", + "choices": [%s], + "model": "%s", + "object": "%s", + "usage": %s + } + """, id, choicesJson, model, object, usageJson); + } else { + return Strings.format(""" + { + "id": "%s", + "choices": [%s], + "model": "%s", + "object": "%s" + } + """, id, choicesJson, model, object); + } + } + + private String createUsageJson(int completionTokens, int promptTokens, int totalTokens) { + return Strings.format(""" + { + "completion_tokens": %d, + "prompt_tokens": %d, + "total_tokens": %d + } + """, completionTokens, promptTokens, totalTokens); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/completion/GoogleAiStudioCompletionRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/completion/GoogleAiStudioCompletionRequestTests.java index 7ffa8940ad6be..065dfee577a82 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/completion/GoogleAiStudioCompletionRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/completion/GoogleAiStudioCompletionRequestTests.java @@ -10,7 +10,7 @@ import org.apache.http.client.methods.HttpPost; import org.elasticsearch.common.Strings; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioCompletionRequest; import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModelTests; @@ -72,7 +72,7 @@ public void testTruncationInfo_ReturnsNull() { assertNull(request.getTruncationInfo()); } - private static DocumentsOnlyInput listOf(String... input) { - return new DocumentsOnlyInput(List.of(input)); + private static ChatCompletionInput listOf(String... input) { + return new ChatCompletionInput(List.of(input)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntityTests.java deleted file mode 100644 index 9d5492f9e9516..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntityTests.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.request.openai; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentType; - -import java.io.IOException; -import java.util.List; - -import static org.hamcrest.CoreMatchers.is; - -public class OpenAiChatCompletionRequestEntityTests extends ESTestCase { - - public void testXContent_WritesUserWhenDefined() throws IOException { - var entity = new OpenAiChatCompletionRequestEntity(List.of("abc"), "model", "user", false); - - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); - entity.toXContent(builder, null); - String xContentResult = Strings.toString(builder); - - assertThat(xContentResult, is(""" - {"messages":[{"role":"user","content":"abc"}],"model":"model","n":1,"user":"user"}""")); - - } - - public void testXContent_DoesNotWriteUserWhenItIsNull() throws IOException { - var entity = new OpenAiChatCompletionRequestEntity(List.of("abc"), "model", null, false); - - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); - entity.toXContent(builder, null); - String xContentResult = Strings.toString(builder); - - assertThat(xContentResult, is(""" - {"messages":[{"role":"user","content":"abc"}],"model":"model","n":1}""")); - } - - public void testXContent_ThrowsIfModelIsNull() { - assertThrows(NullPointerException.class, () -> new OpenAiChatCompletionRequestEntity(List.of("abc"), null, "user", false)); - } - - public void testXContent_ThrowsIfMessagesAreNull() { - assertThrows(NullPointerException.class, () -> new OpenAiChatCompletionRequestEntity(null, "model", "user", false)); - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntityTests.java new file mode 100644 index 0000000000000..f945c154ea234 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestEntityTests.java @@ -0,0 +1,856 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.openai; + +import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.Strings; +import org.elasticsearch.inference.UnifiedCompletionRequest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; +import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Random; + +import static org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModelTests.createChatCompletionModel; +import static org.hamcrest.Matchers.equalTo; + +public class OpenAiUnifiedChatCompletionRequestEntityTests extends ESTestCase { + + // 1. Basic Serialization + // Test with minimal required fields to ensure basic serialization works. + public void testBasicSerialization() throws IOException { + UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString("Hello, world!"), + OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD, + null, + null, + null + ); + var messageList = new ArrayList(); + messageList.add(message); + UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest(messageList, null, null, null, null, null, null, null); + + UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true); + OpenAiChatCompletionModel model = createChatCompletionModel("test-url", "organizationId", "api-key", "test-endpoint", null); + + OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model); + + XContentBuilder builder = JsonXContent.contentBuilder(); + entity.toXContent(builder, ToXContent.EMPTY_PARAMS); + + String jsonString = Strings.toString(builder); + String expectedJson = """ + { + "messages": [ + { + "content": "Hello, world!", + "role": "user" + } + ], + "model": "test-endpoint", + "n": 1, + "stream": true, + "stream_options": { + "include_usage": true + } + } + """; + assertJsonEquals(jsonString, expectedJson); + } + + // 2. Serialization with All Fields + // Test with all possible fields populated to ensure complete serialization. + public void testSerializationWithAllFields() throws IOException { + // Create a message with all fields populated + UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString("Hello, world!"), + OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD, + "name", + "tool_call_id", + Collections.singletonList( + new UnifiedCompletionRequest.ToolCall( + "id", + new UnifiedCompletionRequest.ToolCall.FunctionField("arguments", "function_name"), + "type" + ) + ) + ); + + // Create a tool with all fields populated + UnifiedCompletionRequest.Tool tool = new UnifiedCompletionRequest.Tool( + "type", + new UnifiedCompletionRequest.Tool.FunctionField( + "Fetches the weather in the given location", + "get_weather", + createParameters(), + true + ) + ); + var messageList = new ArrayList(); + messageList.add(message); + // Create the unified request with all fields populated + UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest( + messageList, + "model", + 100L, // maxCompletionTokens + Collections.singletonList("stop"), + 0.9f, // temperature + new UnifiedCompletionRequest.ToolChoiceString("tool_choice"), + Collections.singletonList(tool), + 0.8f // topP + ); + + // Create the unified chat input + UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true); + + OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null); + + // Create the entity + OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model); + + // Serialize to XContent + XContentBuilder builder = JsonXContent.contentBuilder(); + entity.toXContent(builder, ToXContent.EMPTY_PARAMS); + + // Convert to string and verify + String jsonString = Strings.toString(builder); + String expectedJson = """ + { + "messages": [ + { + "content": "Hello, world!", + "role": "user", + "name": "name", + "tool_call_id": "tool_call_id", + "tool_calls": [ + { + "id": "id", + "function": { + "arguments": "arguments", + "name": "function_name" + }, + "type": "type" + } + ] + } + ], + "model": "model-name", + "max_completion_tokens": 100, + "n": 1, + "stop": ["stop"], + "temperature": 0.9, + "tool_choice": "tool_choice", + "tools": [ + { + "type": "type", + "function": { + "description": "Fetches the weather in the given location", + "name": "get_weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "description": "The location to get the weather for", + "type": "string" + }, + "unit": { + "description": "The unit to return the temperature in", + "type": "string", + "enum": ["F", "C"] + } + }, + "additionalProperties": false, + "required": ["location", "unit"] + }, + "strict": true + } + } + ], + "top_p": 0.8, + "stream": true, + "stream_options": { + "include_usage": true + } + } + """; + assertJsonEquals(jsonString, expectedJson); + + } + + // 3. Serialization with Null Optional Fields + // Test with optional fields set to null to ensure they are correctly omitted from the output. + public void testSerializationWithNullOptionalFields() throws IOException { + // Create a message with minimal required fields + UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString("Hello, world!"), + OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD, + null, + null, + null + ); + var messageList = new ArrayList(); + messageList.add(message); + + // Create the unified request with optional fields set to null + UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest( + messageList, + null, // model + null, // maxCompletionTokens + null, // stop + null, // temperature + null, // toolChoice + null, // tools + null // topP + ); + + // Create the unified chat input + UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true); + + OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null); + + // Create the entity + OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model); + + // Serialize to XContent + XContentBuilder builder = JsonXContent.contentBuilder(); + entity.toXContent(builder, ToXContent.EMPTY_PARAMS); + + // Convert to string and verify + String jsonString = Strings.toString(builder); + String expectedJson = """ + { + "messages": [ + { + "content": "Hello, world!", + "role": "user" + } + ], + "model": "model-name", + "n": 1, + "stream": true, + "stream_options": { + "include_usage": true + } + } + """; + assertJsonEquals(jsonString, expectedJson); + } + + // 4. Serialization with Empty Lists + // Test with fields that are lists set to empty lists to ensure they are correctly serialized. + public void testSerializationWithEmptyLists() throws IOException { + // Create a message with minimal required fields + UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString("Hello, world!"), + OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD, + null, + null, + Collections.emptyList() // empty toolCalls list + ); + var messageList = new ArrayList(); + messageList.add(message); + // Create the unified request with empty lists + UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest( + messageList, + null, // model + null, // maxCompletionTokens + Collections.emptyList(), // empty stop list + null, // temperature + null, // toolChoice + Collections.emptyList(), // empty tools list + null // topP + ); + + // Create the unified chat input + UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true); + + OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null); + + // Create the entity + OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model); + + // Serialize to XContent + XContentBuilder builder = JsonXContent.contentBuilder(); + entity.toXContent(builder, ToXContent.EMPTY_PARAMS); + + // Convert to string and verify + String jsonString = Strings.toString(builder); + String expectedJson = """ + { + "messages": [ + { + "content": "Hello, world!", + "role": "user", + "tool_calls": [] + } + ], + "model": "model-name", + "n": 1, + "stream": true, + "stream_options": { + "include_usage": true + } + } + """; + assertJsonEquals(jsonString, expectedJson); + } + + // 5. Serialization with Nested Objects + // Test with nested objects (e.g., toolCalls, toolChoice, tool) to ensure they are correctly serialized. + public void testSerializationWithNestedObjects() throws IOException { + Random random = Randomness.get(); + + // Generate random values + String randomContent = "Hello, world! " + random.nextInt(1000); + String randomName = "name" + random.nextInt(1000); + String randomToolCallId = "tool_call_id" + random.nextInt(1000); + String randomArguments = "arguments" + random.nextInt(1000); + String randomFunctionName = "function_name" + random.nextInt(1000); + String randomType = "type" + random.nextInt(1000); + String randomModel = "model" + random.nextInt(1000); + String randomStop = "stop" + random.nextInt(1000); + float randomTemperature = (float) ((float) Math.round(0.5d + (double) random.nextFloat() * 0.5d * 100000d) / 100000d); + float randomTopP = (float) ((float) Math.round(0.5d + (double) random.nextFloat() * 0.5d * 100000d) / 100000d); + + // Create a message with nested toolCalls + UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString(randomContent), + OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD, + randomName, + randomToolCallId, + Collections.singletonList( + new UnifiedCompletionRequest.ToolCall( + "id", + new UnifiedCompletionRequest.ToolCall.FunctionField(randomArguments, randomFunctionName), + randomType + ) + ) + ); + + // Create a tool with nested function fields + UnifiedCompletionRequest.Tool tool = new UnifiedCompletionRequest.Tool( + randomType, + new UnifiedCompletionRequest.Tool.FunctionField( + "Fetches the weather in the given location", + "get_weather", + createParameters(), + true + ) + ); + var messageList = new ArrayList(); + messageList.add(message); + // Create the unified request with nested objects + UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest( + messageList, + randomModel, + 100L, // maxCompletionTokens + Collections.singletonList(randomStop), + randomTemperature, // temperature + new UnifiedCompletionRequest.ToolChoiceObject( + randomType, + new UnifiedCompletionRequest.ToolChoiceObject.FunctionField(randomFunctionName) + ), + Collections.singletonList(tool), + randomTopP // topP + ); + + // Create the unified chat input + UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true); + + OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", randomModel, null); + + // Create the entity + OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model); + + // Serialize to XContent + XContentBuilder builder = JsonXContent.contentBuilder(); + entity.toXContent(builder, ToXContent.EMPTY_PARAMS); + + // Convert to string and verify + String jsonString = Strings.toString(builder); + // Expected JSON should be dynamically generated based on random values + String expectedJson = String.format( + Locale.US, + """ + { + "messages": [ + { + "content": "%s", + "role": "user", + "name": "%s", + "tool_call_id": "%s", + "tool_calls": [ + { + "id": "id", + "function": { + "arguments": "%s", + "name": "%s" + }, + "type": "%s" + } + ] + } + ], + "model": "%s", + "max_completion_tokens": 100, + "n": 1, + "stop": ["%s"], + "temperature": %.5f, + "tool_choice": { + "type": "%s", + "function": { + "name": "%s" + } + }, + "tools": [ + { + "type": "%s", + "function": { + "description": "Fetches the weather in the given location", + "name": "get_weather", + "parameters": { + "type": "object", + "properties": { + "unit": { + "description": "The unit to return the temperature in", + "type": "string", + "enum": ["F", "C"] + }, + "location": { + "description": "The location to get the weather for", + "type": "string" + } + }, + "additionalProperties": false, + "required": ["location", "unit"] + }, + "strict": true + } + } + ], + "top_p": %.5f, + "stream": true, + "stream_options": { + "include_usage": true + } + } + """, + randomContent, + randomName, + randomToolCallId, + randomArguments, + randomFunctionName, + randomType, + randomModel, + randomStop, + randomTemperature, + randomType, + randomFunctionName, + randomType, + randomTopP + ); + assertJsonEquals(jsonString, expectedJson); + } + + // 6. Serialization with Different Content Types + // Test with different content types in messages (e.g., ContentString, ContentObjects) to ensure they are correctly serialized. + public void testSerializationWithDifferentContentTypes() throws IOException { + Random random = Randomness.get(); + + // Generate random values for ContentString + String randomContentString = "Hello, world! " + random.nextInt(1000); + + // Generate random values for ContentObjects + String randomText = "Random text " + random.nextInt(1000); + String randomType = "type" + random.nextInt(1000); + UnifiedCompletionRequest.ContentObject contentObject = new UnifiedCompletionRequest.ContentObject(randomText, randomType); + + var contentObjectsList = new ArrayList(); + contentObjectsList.add(contentObject); + UnifiedCompletionRequest.ContentObjects contentObjects = new UnifiedCompletionRequest.ContentObjects(contentObjectsList); + + // Create messages with different content types + UnifiedCompletionRequest.Message messageWithString = new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString(randomContentString), + OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD, + null, + null, + null + ); + + UnifiedCompletionRequest.Message messageWithObjects = new UnifiedCompletionRequest.Message( + contentObjects, + OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD, + null, + null, + null + ); + var messageList = new ArrayList(); + messageList.add(messageWithString); + messageList.add(messageWithObjects); + + // Create the unified request with both types of messages + UnifiedCompletionRequest unifiedRequest = UnifiedCompletionRequest.of(messageList); + + // Create the unified chat input + UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true); + + OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null); + + // Create the entity + OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model); + + // Serialize to XContent + XContentBuilder builder = JsonXContent.contentBuilder(); + entity.toXContent(builder, ToXContent.EMPTY_PARAMS); + + // Convert to string and verify + String jsonString = Strings.toString(builder); + String expectedJson = String.format(Locale.US, """ + { + "messages": [ + { + "content": "%s", + "role": "user" + }, + { + "content": [ + { + "text": "%s", + "type": "%s" + } + ], + "role": "user" + } + ], + "model": "model-name", + "n": 1, + "stream": true, + "stream_options": { + "include_usage": true + } + } + """, randomContentString, randomText, randomType); + assertJsonEquals(jsonString, expectedJson); + } + + // 7. Serialization with Special Characters + // Test with special characters in string fields to ensure they are correctly escaped and serialized. + public void testSerializationWithSpecialCharacters() throws IOException { + // Create a message with special characters + UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString("Hello, world! \n \"Special\" characters: \t \\ /"), + OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD, + "name\nwith\nnewlines", + "tool_call_id\twith\ttabs", + Collections.singletonList( + new UnifiedCompletionRequest.ToolCall( + "id\\with\\backslashes", + new UnifiedCompletionRequest.ToolCall.FunctionField("arguments\"with\"quotes", "function_name/with/slashes"), + "type" + ) + ) + ); + var messageList = new ArrayList(); + messageList.add(message); + // Create the unified request + UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest( + messageList, + null, // model + null, // maxCompletionTokens + null, // stop + null, // temperature + null, // toolChoice + null, // tools + null // topP + ); + + // Create the unified chat input + UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true); + + OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null); + + // Create the entity + OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model); + + // Serialize to XContent + XContentBuilder builder = JsonXContent.contentBuilder(); + entity.toXContent(builder, ToXContent.EMPTY_PARAMS); + + // Convert to string and verify + String jsonString = Strings.toString(builder); + String expectedJson = """ + { + "messages": [ + { + "content": "Hello, world! \\n \\"Special\\" characters: \\t \\\\ /", + "role": "user", + "name": "name\\nwith\\nnewlines", + "tool_call_id": "tool_call_id\\twith\\ttabs", + "tool_calls": [ + { + "id": "id\\\\with\\\\backslashes", + "function": { + "arguments": "arguments\\"with\\"quotes", + "name": "function_name/with/slashes" + }, + "type": "type" + } + ] + } + ], + "model": "model-name", + "n": 1, + "stream": true, + "stream_options": { + "include_usage": true + } + } + """; + assertJsonEquals(jsonString, expectedJson); + } + + // 8. Serialization with Boolean Fields + // Test with boolean fields (stream) set to both true and false to ensure they are correctly serialized. + public void testSerializationWithBooleanFields() throws IOException { + // Create a message with minimal required fields + UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString("Hello, world!"), + OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD, + null, + null, + null + ); + var messageList = new ArrayList(); + messageList.add(message); + // Create the unified request + UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest( + messageList, + null, // model + null, // maxCompletionTokens + null, // stop + null, // temperature + null, // toolChoice + null, // tools + null // topP + ); + + OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null); + + // Test with stream set to true + UnifiedChatInput unifiedChatInputTrue = new UnifiedChatInput(unifiedRequest, true); + OpenAiUnifiedChatCompletionRequestEntity entityTrue = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInputTrue, model); + + XContentBuilder builderTrue = JsonXContent.contentBuilder(); + entityTrue.toXContent(builderTrue, ToXContent.EMPTY_PARAMS); + + String jsonStringTrue = Strings.toString(builderTrue); + String expectedJsonTrue = """ + { + "messages": [ + { + "content": "Hello, world!", + "role": "user" + } + ], + "model": "model-name", + "n": 1, + "stream": true, + "stream_options": { + "include_usage": true + } + } + """; + assertJsonEquals(expectedJsonTrue, jsonStringTrue); + + // Test with stream set to false + UnifiedChatInput unifiedChatInputFalse = new UnifiedChatInput(unifiedRequest, false); + OpenAiUnifiedChatCompletionRequestEntity entityFalse = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInputFalse, model); + + XContentBuilder builderFalse = JsonXContent.contentBuilder(); + entityFalse.toXContent(builderFalse, ToXContent.EMPTY_PARAMS); + + String jsonStringFalse = Strings.toString(builderFalse); + String expectedJsonFalse = """ + { + "messages": [ + { + "content": "Hello, world!", + "role": "user" + } + ], + "model": "model-name", + "n": 1, + "stream": false + } + """; + assertJsonEquals(expectedJsonFalse, jsonStringFalse); + } + + // 9. Serialization with Missing Required Fields + // Test with missing required fields to ensure appropriate exceptions are thrown. + public void testSerializationWithMissingRequiredFields() { + // Create a message with missing content (required field) + UnifiedCompletionRequest.Message message = new UnifiedCompletionRequest.Message( + null, // missing content + OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD, + null, + null, + null + ); + var messageList = new ArrayList(); + messageList.add(message); + // Create the unified request + UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest( + messageList, + null, // model + null, // maxCompletionTokens + null, // stop + null, // temperature + null, // toolChoice + null, // tools + null // topP + ); + + // Create the unified chat input + UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true); + + OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null); + + // Create the entity + OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model); + + // Attempt to serialize to XContent and expect an exception + try { + XContentBuilder builder = JsonXContent.contentBuilder(); + entity.toXContent(builder, ToXContent.EMPTY_PARAMS); + fail("Expected an exception due to missing required fields"); + } catch (NullPointerException | IOException e) { + // Expected exception + } + } + + // 10. Serialization with Mixed Valid and Invalid Data + // Test with a mix of valid and invalid data to ensure the serializer handles it gracefully. + public void testSerializationWithMixedValidAndInvalidData() throws IOException { + // Create a valid message + UnifiedCompletionRequest.Message validMessage = new UnifiedCompletionRequest.Message( + new UnifiedCompletionRequest.ContentString("Valid content"), + OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD, + "validName", + "validToolCallId", + Collections.singletonList( + new UnifiedCompletionRequest.ToolCall( + "validId", + new UnifiedCompletionRequest.ToolCall.FunctionField("validArguments", "validFunctionName"), + "validType" + ) + ) + ); + + // Create an invalid message with null content + UnifiedCompletionRequest.Message invalidMessage = new UnifiedCompletionRequest.Message( + null, // invalid content + OpenAiUnifiedChatCompletionRequestEntity.USER_FIELD, + "invalidName", + "invalidToolCallId", + Collections.singletonList( + new UnifiedCompletionRequest.ToolCall( + "invalidId", + new UnifiedCompletionRequest.ToolCall.FunctionField("invalidArguments", "invalidFunctionName"), + "invalidType" + ) + ) + ); + var messageList = new ArrayList(); + messageList.add(validMessage); + messageList.add(invalidMessage); + // Create the unified request with both valid and invalid messages + UnifiedCompletionRequest unifiedRequest = new UnifiedCompletionRequest( + messageList, + "model-name", + 100L, // maxCompletionTokens + Collections.singletonList("stop"), + 0.9f, // temperature + new UnifiedCompletionRequest.ToolChoiceString("tool_choice"), + Collections.singletonList( + new UnifiedCompletionRequest.Tool( + "type", + new UnifiedCompletionRequest.Tool.FunctionField( + "Fetches the weather in the given location", + "get_weather", + createParameters(), + true + ) + ) + ), + 0.8f // topP + ); + + // Create the unified chat input + UnifiedChatInput unifiedChatInput = new UnifiedChatInput(unifiedRequest, true); + + OpenAiChatCompletionModel model = createChatCompletionModel("test-endpoint", "organizationId", "api-key", "model-name", null); + + // Create the entity + OpenAiUnifiedChatCompletionRequestEntity entity = new OpenAiUnifiedChatCompletionRequestEntity(unifiedChatInput, model); + + // Serialize to XContent and verify + try { + XContentBuilder builder = JsonXContent.contentBuilder(); + entity.toXContent(builder, ToXContent.EMPTY_PARAMS); + fail("Expected an exception due to invalid data"); + } catch (NullPointerException | IOException e) { + // Expected exception + } + } + + public static Map createParameters() { + Map parameters = new LinkedHashMap<>(); + parameters.put("type", "object"); + + Map properties = new HashMap<>(); + + Map location = new HashMap<>(); + location.put("type", "string"); + location.put("description", "The location to get the weather for"); + properties.put("location", location); + + Map unit = new HashMap<>(); + unit.put("type", "string"); + unit.put("description", "The unit to return the temperature in"); + unit.put("enum", new String[] { "F", "C" }); + properties.put("unit", unit); + + parameters.put("properties", properties); + parameters.put("additionalProperties", false); + parameters.put("required", new String[] { "location", "unit" }); + + return parameters; + } + + private void assertJsonEquals(String actual, String expected) throws IOException { + try ( + var actualParser = createParser(JsonXContent.jsonXContent, actual); + var expectedParser = createParser(JsonXContent.jsonXContent, expected) + ) { + assertThat(actualParser.mapOrdered(), equalTo(expectedParser.mapOrdered())); + } + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestTests.java similarity index 75% rename from x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestTests.java rename to x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestTests.java index b6ebfd02941f3..2be12c9b12e0b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiUnifiedChatCompletionRequestTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModelTests; import java.io.IOException; @@ -20,16 +21,16 @@ import java.util.Map; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; -import static org.elasticsearch.xpack.inference.external.request.openai.OpenAiChatCompletionRequest.buildDefaultUri; +import static org.elasticsearch.xpack.inference.external.request.openai.OpenAiUnifiedChatCompletionRequest.buildDefaultUri; import static org.elasticsearch.xpack.inference.external.request.openai.OpenAiUtils.ORGANIZATION_HEADER; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -public class OpenAiChatCompletionRequestTests extends ESTestCase { +public class OpenAiUnifiedChatCompletionRequestTests extends ESTestCase { public void testCreateRequest_WithUrlOrganizationUserDefined() throws IOException { - var request = createRequest("www.google.com", "org", "secret", "abc", "model", "user"); + var request = createRequest("www.google.com", "org", "secret", "abc", "model", "user", true); var httpRequest = request.createHttpRequest(); assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); @@ -41,15 +42,27 @@ public void testCreateRequest_WithUrlOrganizationUserDefined() throws IOExceptio assertThat(httpPost.getLastHeader(ORGANIZATION_HEADER).getValue(), is("org")); var requestMap = entityAsMap(httpPost.getEntity().getContent()); - assertThat(requestMap, aMapWithSize(4)); + assertRequestMapWithUser(requestMap, "user"); + } + + private void assertRequestMapWithoutUser(Map requestMap) { + assertRequestMapWithUser(requestMap, null); + } + + private void assertRequestMapWithUser(Map requestMap, @Nullable String user) { + assertThat(requestMap, aMapWithSize(user != null ? 6 : 5)); assertThat(requestMap.get("messages"), is(List.of(Map.of("role", "user", "content", "abc")))); assertThat(requestMap.get("model"), is("model")); - assertThat(requestMap.get("user"), is("user")); + if (user != null) { + assertThat(requestMap.get("user"), is(user)); + } assertThat(requestMap.get("n"), is(1)); + assertTrue((Boolean) requestMap.get("stream")); + assertThat(requestMap.get("stream_options"), is(Map.of("include_usage", true))); } public void testCreateRequest_WithDefaultUrl() throws URISyntaxException, IOException { - var request = createRequest(null, "org", "secret", "abc", "model", "user"); + var request = createRequest(null, "org", "secret", "abc", "model", "user", true); var httpRequest = request.createHttpRequest(); assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); @@ -61,33 +74,27 @@ public void testCreateRequest_WithDefaultUrl() throws URISyntaxException, IOExce assertThat(httpPost.getLastHeader(ORGANIZATION_HEADER).getValue(), is("org")); var requestMap = entityAsMap(httpPost.getEntity().getContent()); - assertThat(requestMap, aMapWithSize(4)); - assertThat(requestMap.get("messages"), is(List.of(Map.of("role", "user", "content", "abc")))); - assertThat(requestMap.get("model"), is("model")); - assertThat(requestMap.get("user"), is("user")); - assertThat(requestMap.get("n"), is(1)); + assertRequestMapWithUser(requestMap, "user"); + } public void testCreateRequest_WithDefaultUrlAndWithoutUserOrganization() throws URISyntaxException, IOException { - var request = createRequest(null, null, "secret", "abc", "model", null); + var request = createRequest(null, null, "secret", "abc", "model", null, true); var httpRequest = request.createHttpRequest(); assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); var httpPost = (HttpPost) httpRequest.httpRequestBase(); - assertThat(httpPost.getURI().toString(), is(OpenAiChatCompletionRequest.buildDefaultUri().toString())); + assertThat(httpPost.getURI().toString(), is(OpenAiUnifiedChatCompletionRequest.buildDefaultUri().toString())); assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret")); assertNull(httpPost.getLastHeader(ORGANIZATION_HEADER)); var requestMap = entityAsMap(httpPost.getEntity().getContent()); - assertThat(requestMap, aMapWithSize(3)); - assertThat(requestMap.get("messages"), is(List.of(Map.of("role", "user", "content", "abc")))); - assertThat(requestMap.get("model"), is("model")); - assertThat(requestMap.get("n"), is(1)); + assertRequestMapWithoutUser(requestMap); } - public void testCreateRequest_WithStreaming() throws URISyntaxException, IOException { + public void testCreateRequest_WithStreaming() throws IOException { var request = createRequest(null, null, "secret", "abc", "model", null, true); var httpRequest = request.createHttpRequest(); @@ -99,29 +106,31 @@ public void testCreateRequest_WithStreaming() throws URISyntaxException, IOExcep } public void testTruncate_DoesNotReduceInputTextSize() throws URISyntaxException, IOException { - var request = createRequest(null, null, "secret", "abcd", "model", null); + var request = createRequest(null, null, "secret", "abcd", "model", null, true); var truncatedRequest = request.truncate(); - assertThat(request.getURI().toString(), is(OpenAiChatCompletionRequest.buildDefaultUri().toString())); + assertThat(request.getURI().toString(), is(OpenAiUnifiedChatCompletionRequest.buildDefaultUri().toString())); var httpRequest = truncatedRequest.createHttpRequest(); assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); var httpPost = (HttpPost) httpRequest.httpRequestBase(); var requestMap = entityAsMap(httpPost.getEntity().getContent()); - assertThat(requestMap, aMapWithSize(3)); + assertThat(requestMap, aMapWithSize(5)); // We do not truncate for OpenAi chat completions assertThat(requestMap.get("messages"), is(List.of(Map.of("role", "user", "content", "abcd")))); assertThat(requestMap.get("model"), is("model")); assertThat(requestMap.get("n"), is(1)); + assertTrue((Boolean) requestMap.get("stream")); + assertThat(requestMap.get("stream_options"), is(Map.of("include_usage", true))); } public void testTruncationInfo_ReturnsNull() { - var request = createRequest(null, null, "secret", "abcd", "model", null); + var request = createRequest(null, null, "secret", "abcd", "model", null, true); assertNull(request.getTruncationInfo()); } - public static OpenAiChatCompletionRequest createRequest( + public static OpenAiUnifiedChatCompletionRequest createRequest( @Nullable String url, @Nullable String org, String apiKey, @@ -132,7 +141,7 @@ public static OpenAiChatCompletionRequest createRequest( return createRequest(url, org, apiKey, input, model, user, false); } - public static OpenAiChatCompletionRequest createRequest( + public static OpenAiUnifiedChatCompletionRequest createRequest( @Nullable String url, @Nullable String org, String apiKey, @@ -142,7 +151,7 @@ public static OpenAiChatCompletionRequest createRequest( boolean stream ) { var chatCompletionModel = OpenAiChatCompletionModelTests.createChatCompletionModel(url, org, apiKey, model, user); - return new OpenAiChatCompletionRequest(List.of(input), chatCompletionModel, stream); + return new OpenAiUnifiedChatCompletionRequest(new UnifiedChatInput(List.of(input), "user", stream), chatCompletionModel); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/BaseInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/BaseInferenceActionTests.java index 05a8d52be5df4..5528c80066b0a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/BaseInferenceActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/BaseInferenceActionTests.java @@ -8,11 +8,14 @@ package org.elasticsearch.xpack.inference.rest; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestRequestTests; import org.elasticsearch.rest.action.RestChunkedToXContentListener; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.test.rest.RestActionTestCase; @@ -26,6 +29,10 @@ import java.util.Map; import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.xpack.inference.rest.BaseInferenceAction.parseParams; +import static org.elasticsearch.xpack.inference.rest.BaseInferenceAction.parseTimeout; +import static org.elasticsearch.xpack.inference.rest.Paths.INFERENCE_ID; +import static org.elasticsearch.xpack.inference.rest.Paths.TASK_TYPE_OR_INFERENCE_ID; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -56,6 +63,42 @@ private static String route(String param) { return "_route/" + param; } + public void testParseParams_ExtractsInferenceIdAndTaskType() { + var params = parseParams( + RestRequestTests.contentRestRequest("{}", Map.of(INFERENCE_ID, "id", TASK_TYPE_OR_INFERENCE_ID, TaskType.COMPLETION.toString())) + ); + assertThat(params, is(new BaseInferenceAction.Params("id", TaskType.COMPLETION))); + } + + public void testParseParams_DefaultsToTaskTypeAny_WhenInferenceId_IsMissing() { + var params = parseParams( + RestRequestTests.contentRestRequest("{}", Map.of(TASK_TYPE_OR_INFERENCE_ID, TaskType.COMPLETION.toString())) + ); + assertThat(params, is(new BaseInferenceAction.Params("completion", TaskType.ANY))); + } + + public void testParseParams_ThrowsStatusException_WhenTaskTypeIsMissing() { + var e = expectThrows( + ElasticsearchStatusException.class, + () -> parseParams(RestRequestTests.contentRestRequest("{}", Map.of(INFERENCE_ID, "id"))) + ); + assertThat(e.getMessage(), is("Task type must not be null")); + } + + public void testParseTimeout_ReturnsTimeout() { + var timeout = parseTimeout( + RestRequestTests.contentRestRequest("{}", Map.of(InferenceAction.Request.TIMEOUT.getPreferredName(), "4s")) + ); + + assertThat(timeout, is(TimeValue.timeValueSeconds(4))); + } + + public void testParseTimeout_ReturnsDefaultTimeout() { + var timeout = parseTimeout(RestRequestTests.contentRestRequest("{}", Map.of())); + + assertThat(timeout, is(TimeValue.timeValueSeconds(30))); + } + public void testUsesDefaultTimeout() { SetOnce executeCalled = new SetOnce<>(); verifyingClient.setExecuteVerifier(((actionType, actionRequest) -> { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceActionTests.java new file mode 100644 index 0000000000000..5acfe67b175df --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestUnifiedCompletionInferenceActionTests.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.rest; + +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.rest.AbstractRestChannel; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.test.rest.RestActionTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; +import org.junit.Before; + +import static org.elasticsearch.xpack.inference.rest.BaseInferenceActionTests.createResponse; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class RestUnifiedCompletionInferenceActionTests extends RestActionTestCase { + + @Before + public void setUpAction() { + controller().registerHandler(new RestUnifiedCompletionInferenceAction()); + } + + public void testStreamIsTrue() { + SetOnce executeCalled = new SetOnce<>(); + verifyingClient.setExecuteVerifier(((actionType, actionRequest) -> { + assertThat(actionRequest, instanceOf(UnifiedCompletionAction.Request.class)); + + var request = (UnifiedCompletionAction.Request) actionRequest; + assertThat(request.isStreaming(), is(true)); + + executeCalled.set(true); + return createResponse(); + })); + + var requestBody = """ + { + "messages": [ + { + "content": "abc", + "role": "user" + } + ] + } + """; + + RestRequest inferenceRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST) + .withPath("_inference/completion/test/_unified") + .withContent(new BytesArray(requestBody), XContentType.JSON) + .build(); + + final SetOnce responseSetOnce = new SetOnce<>(); + dispatchRequest(inferenceRequest, new AbstractRestChannel(inferenceRequest, true) { + @Override + public void sendResponse(RestResponse response) { + responseSetOnce.set(response); + } + }); + + // the response content will be null when there is no error + assertNull(responseSetOnce.get().content()); + assertThat(executeCalled.get(), equalTo(true)); + } + + private void dispatchRequest(final RestRequest request, final RestChannel channel) { + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + controller().dispatchRequest(request, channel, threadContext); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java index 47a96bf78dda1..6768583598b2d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; import org.junit.After; import org.junit.Before; @@ -119,6 +120,14 @@ protected void doInfer( } + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) {} + @Override protected void doChunkedInfer( Model model, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java index 76b5d6fee2c59..159b77789482d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.inference.Model; import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnifiedCompletionRequest; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.http.MockResponse; import org.elasticsearch.test.http.MockWebServer; @@ -920,6 +921,68 @@ public void testInfer_SendsRequest() throws IOException { } } + public void testUnifiedCompletionInfer() throws Exception { + // The escapes are because the streaming response must be on a single line + String responseJson = """ + data: {\ + "id":"12345",\ + "object":"chat.completion.chunk",\ + "created":123456789,\ + "model":"gpt-4o-mini",\ + "system_fingerprint": "123456789",\ + "choices":[\ + {\ + "index":0,\ + "delta":{\ + "content":"hello, world"\ + },\ + "logprobs":null,\ + "finish_reason":"stop"\ + }\ + ],\ + "usage":{\ + "prompt_tokens": 16,\ + "completion_tokens": 28,\ + "total_tokens": 44,\ + "prompt_tokens_details": {\ + "cached_tokens": 0,\ + "audio_tokens": 0\ + },\ + "completion_tokens_details": {\ + "reasoning_tokens": 0,\ + "audio_tokens": 0,\ + "accepted_prediction_tokens": 0,\ + "rejected_prediction_tokens": 0\ + }\ + }\ + } + + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new OpenAiService(senderFactory, createWithEmptySettings(threadPool))) { + var model = OpenAiChatCompletionModelTests.createChatCompletionModel(getUrl(webServer), "org", "secret", "model", "user"); + PlainActionFuture listener = new PlainActionFuture<>(); + service.unifiedCompletionInfer( + model, + UnifiedCompletionRequest.of( + List.of( + new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "user", null, null, null) + ) + ), + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + InferenceEventsAssertion.assertThat(result).hasFinishedStream().hasNoErrors().hasEvent(""" + {"id":"12345","choices":[{"delta":{"content":"hello, world"},"finish_reason":"stop","index":0}],""" + """ + "model":"gpt-4o-mini","object":"chat.completion.chunk",""" + """ + "usage":{"completion_tokens":28,"prompt_tokens":16,"total_tokens":44}}"""); + } + } + public void testInfer_StreamRequest() throws Exception { String responseJson = """ data: {\ diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModelTests.java index ab1786f0a5843..e7ac4cf879e92 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModelTests.java @@ -10,9 +10,11 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnifiedCompletionRequest; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import java.util.List; import java.util.Map; import static org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionRequestTaskSettingsTests.getChatCompletionRequestTaskSettingsMap; @@ -42,10 +44,48 @@ public void testOverrideWith_EmptyMap() { public void testOverrideWith_NullMap() { var model = createChatCompletionModel("url", "org", "api_key", "model_name", null); - var overriddenModel = OpenAiChatCompletionModel.of(model, null); + var overriddenModel = OpenAiChatCompletionModel.of(model, (Map) null); assertThat(overriddenModel, sameInstance(model)); } + public void testOverrideWith_UnifiedCompletionRequest_OverridesModelId() { + var model = createChatCompletionModel("url", "org", "api_key", "model_name", "user"); + var request = new UnifiedCompletionRequest( + List.of(new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "role", null, null, null)), + "different_model", + null, + null, + null, + null, + null, + null + ); + + assertThat( + OpenAiChatCompletionModel.of(model, request), + is(createChatCompletionModel("url", "org", "api_key", "different_model", "user")) + ); + } + + public void testOverrideWith_UnifiedCompletionRequest_UsesModelFields_WhenRequestDoesNotOverride() { + var model = createChatCompletionModel("url", "org", "api_key", "model_name", "user"); + var request = new UnifiedCompletionRequest( + List.of(new UnifiedCompletionRequest.Message(new UnifiedCompletionRequest.ContentString("hello"), "role", null, null, null)), + null, // not overriding model + null, + null, + null, + null, + null, + null + ); + + assertThat( + OpenAiChatCompletionModel.of(model, request), + is(createChatCompletionModel("url", "org", "api_key", "model_name", "user")) + ); + } + public static OpenAiChatCompletionModel createChatCompletionModel( String url, @Nullable String org, diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 8df10037affdb..c91314716cf9e 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -386,6 +386,7 @@ public class Constants { "cluster:monitor/xpack/esql/stats/dist", "cluster:monitor/xpack/inference", "cluster:monitor/xpack/inference/get", + "cluster:monitor/xpack/inference/unified", "cluster:monitor/xpack/inference/diagnostics/get", "cluster:monitor/xpack/inference/services/get", "cluster:monitor/xpack/info", From 4df28327efddc8864590d33eb68105146b8b7754 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 7 Dec 2024 07:55:27 +1100 Subject: [PATCH 095/204] Mute org.elasticsearch.xpack.esql.action.EsqlActionTaskIT testCancelRequestWhenFailingFetchingPages #118193 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index a39265756599d..4431d3661eb01 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -263,6 +263,9 @@ tests: - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT method: testInferDeploysDefaultRerank issue: https://github.com/elastic/elasticsearch/issues/118184 +- class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT + method: testCancelRequestWhenFailingFetchingPages + issue: https://github.com/elastic/elasticsearch/issues/118193 # Examples: # From c9feb7690bfe880784406d4ad32ca7d1547d8cfe Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 7 Dec 2024 08:17:11 +1100 Subject: [PATCH 096/204] Mute org.elasticsearch.packaging.test.MemoryLockingTests test20MemoryLockingEnabled #118195 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 4431d3661eb01..22a53673dcb34 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -266,6 +266,9 @@ tests: - class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT method: testCancelRequestWhenFailingFetchingPages issue: https://github.com/elastic/elasticsearch/issues/118193 +- class: org.elasticsearch.packaging.test.MemoryLockingTests + method: test20MemoryLockingEnabled + issue: https://github.com/elastic/elasticsearch/issues/118195 # Examples: # From 287ed8a1c177a98e7397463e9a02794fed8b2761 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 7 Dec 2024 08:17:26 +1100 Subject: [PATCH 097/204] Mute org.elasticsearch.packaging.test.ArchiveTests test42AutoconfigurationNotTriggeredWhenNodeCannotBecomeMaster #118196 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 22a53673dcb34..7c5df966e6bd6 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -269,6 +269,9 @@ tests: - class: org.elasticsearch.packaging.test.MemoryLockingTests method: test20MemoryLockingEnabled issue: https://github.com/elastic/elasticsearch/issues/118195 +- class: org.elasticsearch.packaging.test.ArchiveTests + method: test42AutoconfigurationNotTriggeredWhenNodeCannotBecomeMaster + issue: https://github.com/elastic/elasticsearch/issues/118196 # Examples: # From 05eee61038693a11ed8fc8ea5c5a66d460d5c0cf Mon Sep 17 00:00:00 2001 From: Kazuma Arimura Date: Sat, 7 Dec 2024 02:00:03 +0000 Subject: [PATCH 098/204] add documentation for kuromoji_completion (#117808) --- docs/plugins/analysis-kuromoji.asciidoc | 36 +++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc index 0a167bf3f0240..217d88f361223 100644 --- a/docs/plugins/analysis-kuromoji.asciidoc +++ b/docs/plugins/analysis-kuromoji.asciidoc @@ -750,3 +750,39 @@ Which results in: ] } -------------------------------------------------- + +[[analysis-kuromoji-completion]] +==== `kuromoji_completion` token filter + +The `kuromoji_completion` token filter adds Japanese romanized tokens to the term attributes along with the original tokens (surface forms). + +[source,console] +-------------------------------------------------- +GET _analyze +{ + "analyzer": "kuromoji_completion", + "text": "寿司" <1> +} +-------------------------------------------------- + +<1> Returns `寿司`, `susi` (Kunrei-shiki) and `sushi` (Hepburn-shiki). + +The `kuromoji_completion` token filter accepts the following settings: + +`mode`:: ++ +-- + +The tokenization mode determines how the tokenizer handles compound and +unknown words. It can be set to: + +`index`:: + + Simple romanization. Expected to be used when indexing. + +`query`:: + + Input Method aware romanization. Expected to be used when querying. + +Defaults to `index`. +-- From 39c7e0bc2fa6345cde605257c80182a83c2cc951 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 6 Dec 2024 18:24:08 -0800 Subject: [PATCH 099/204] Always write unicast hosts file in tests (#118121) This commit adds the ability to restart a node within a test cluster. The newly started node uses the same configuration and directories. Since we use ephemeral ports, the unicast hosts list is rewritten for all nodes once the restarted node comes back up. --- .../test/cluster/local/DefaultLocalClusterHandle.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java index eb45aacda68da..13adde1da8a69 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java @@ -176,8 +176,9 @@ public long getPid(int index) { return nodes.get(index).getPid(); } + @Override public void stopNode(int index, boolean forcibly) { - nodes.get(index).stop(false); + nodes.get(index).stop(forcibly); } @Override @@ -252,9 +253,8 @@ private void writeUnicastHostsFile() { execute(() -> nodes.parallelStream().forEach(node -> { try { Path hostsFile = node.getWorkingDir().resolve("config").resolve("unicast_hosts.txt"); - if (Files.notExists(hostsFile)) { - Files.writeString(hostsFile, transportUris); - } + LOGGER.info("Writing unicast hosts file {} for node {}", hostsFile, node.getName()); + Files.writeString(hostsFile, transportUris); } catch (IOException e) { throw new UncheckedIOException("Failed to write unicast_hosts for: " + node, e); } From bc0b77e147e2baa84f9d920c1f9604211f6e3ab8 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 7 Dec 2024 16:17:38 +1100 Subject: [PATCH 100/204] Mute org.elasticsearch.packaging.test.ArchiveTests test43AutoconfigurationNotTriggeredWhenTlsAlreadyConfigured #118202 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 7c5df966e6bd6..05f6e0b57415d 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -272,6 +272,9 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveTests method: test42AutoconfigurationNotTriggeredWhenNodeCannotBecomeMaster issue: https://github.com/elastic/elasticsearch/issues/118196 +- class: org.elasticsearch.packaging.test.ArchiveTests + method: test43AutoconfigurationNotTriggeredWhenTlsAlreadyConfigured + issue: https://github.com/elastic/elasticsearch/issues/118202 # Examples: # From 2005e2eec1f53c26b53fcafb58e312fe10b5aae4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sun, 8 Dec 2024 00:16:15 +1100 Subject: [PATCH 101/204] Mute org.elasticsearch.packaging.test.ArchiveTests test44AutoConfigurationNotTriggeredOnNotWriteableConfDir #118208 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 05f6e0b57415d..5914978cba076 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -275,6 +275,9 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveTests method: test43AutoconfigurationNotTriggeredWhenTlsAlreadyConfigured issue: https://github.com/elastic/elasticsearch/issues/118202 +- class: org.elasticsearch.packaging.test.ArchiveTests + method: test44AutoConfigurationNotTriggeredOnNotWriteableConfDir + issue: https://github.com/elastic/elasticsearch/issues/118208 # Examples: # From eb0a21efd8946df04a3f6f0457bb0a1b73b50bf0 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sat, 7 Dec 2024 15:13:56 +0100 Subject: [PATCH 102/204] Speedup OsStats initialization (#118141) Similar to other OS/FS type stats we can optimize here. Found this as a slowdown when profiling tests in a loop during test fixing. This helps node startup and maybe more importantly test performance. No need to initialize the stats eagerly when we can just get them as we load them the first time. --- .../elasticsearch/monitor/os/OsService.java | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsService.java b/server/src/main/java/org/elasticsearch/monitor/os/OsService.java index 7609cc14c6b3b..ceed2b0e41fc1 100644 --- a/server/src/main/java/org/elasticsearch/monitor/os/OsService.java +++ b/server/src/main/java/org/elasticsearch/monitor/os/OsService.java @@ -25,7 +25,6 @@ public class OsService implements ReportingService { private static final Logger logger = LogManager.getLogger(OsService.class); - private final OsProbe probe; private final OsInfo info; private final SingleObjectCache osStatsCache; @@ -37,10 +36,9 @@ public class OsService implements ReportingService { ); public OsService(Settings settings) throws IOException { - this.probe = OsProbe.getInstance(); TimeValue refreshInterval = REFRESH_INTERVAL_SETTING.get(settings); - this.info = probe.osInfo(refreshInterval.millis(), EsExecutors.nodeProcessors(settings)); - this.osStatsCache = new OsStatsCache(refreshInterval, probe.osStats()); + this.info = OsProbe.getInstance().osInfo(refreshInterval.millis(), EsExecutors.nodeProcessors(settings)); + this.osStatsCache = new OsStatsCache(refreshInterval); logger.debug("using refresh_interval [{}]", refreshInterval); } @@ -53,14 +51,28 @@ public OsStats stats() { return osStatsCache.getOrRefresh(); } - private class OsStatsCache extends SingleObjectCache { - OsStatsCache(TimeValue interval, OsStats initValue) { - super(interval, initValue); + private static class OsStatsCache extends SingleObjectCache { + + private static final OsStats MISSING = new OsStats( + 0L, + new OsStats.Cpu((short) 0, new double[0]), + new OsStats.Mem(0, 0, 0), + new OsStats.Swap(0, 0), + null + ); + + OsStatsCache(TimeValue interval) { + super(interval, MISSING); } @Override protected OsStats refresh() { - return probe.osStats(); + return OsProbe.getInstance().osStats(); + } + + @Override + protected boolean needsRefresh() { + return getNoRefresh() == MISSING || super.needsRefresh(); } } } From 230b283beaff21c331c3da9c31d29211e6392d75 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sun, 8 Dec 2024 08:15:30 +1100 Subject: [PATCH 103/204] Mute org.elasticsearch.packaging.test.ArchiveTests test51AutoConfigurationWithPasswordProtectedKeystore #118212 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 5914978cba076..273094134fa2b 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -278,6 +278,9 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveTests method: test44AutoConfigurationNotTriggeredOnNotWriteableConfDir issue: https://github.com/elastic/elasticsearch/issues/118208 +- class: org.elasticsearch.packaging.test.ArchiveTests + method: test51AutoConfigurationWithPasswordProtectedKeystore + issue: https://github.com/elastic/elasticsearch/issues/118212 # Examples: # From 9071e80a923690dcb37369849450da6deb38f95f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sun, 8 Dec 2024 08:55:27 +1100 Subject: [PATCH 104/204] Mute org.elasticsearch.xpack.inference.InferenceCrudIT testUnifiedCompletionInference #118210 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 273094134fa2b..29621676fa836 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -281,6 +281,9 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveTests method: test51AutoConfigurationWithPasswordProtectedKeystore issue: https://github.com/elastic/elasticsearch/issues/118212 +- class: org.elasticsearch.xpack.inference.InferenceCrudIT + method: testUnifiedCompletionInference + issue: https://github.com/elastic/elasticsearch/issues/118210 # Examples: # From 92e0f972fb1795fb8c7e96c0b60aa6f2f2405b83 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sun, 8 Dec 2024 16:25:11 +1100 Subject: [PATCH 105/204] Mute org.elasticsearch.ingest.common.IngestCommonClientYamlTestSuiteIT org.elasticsearch.ingest.common.IngestCommonClientYamlTestSuiteIT #118215 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 29621676fa836..e00bcb068ac26 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -284,6 +284,8 @@ tests: - class: org.elasticsearch.xpack.inference.InferenceCrudIT method: testUnifiedCompletionInference issue: https://github.com/elastic/elasticsearch/issues/118210 +- class: org.elasticsearch.ingest.common.IngestCommonClientYamlTestSuiteIT + issue: https://github.com/elastic/elasticsearch/issues/118215 # Examples: # From 2b0bf196be87bc2b10580f2d89b971b82927cdd6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sun, 8 Dec 2024 16:59:40 +1100 Subject: [PATCH 106/204] Mute org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT test {p0=data_stream/120_data_streams_stats/Multiple data stream} #118217 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e00bcb068ac26..3977b63224f24 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -286,6 +286,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/118210 - class: org.elasticsearch.ingest.common.IngestCommonClientYamlTestSuiteIT issue: https://github.com/elastic/elasticsearch/issues/118215 +- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT + method: test {p0=data_stream/120_data_streams_stats/Multiple data stream} + issue: https://github.com/elastic/elasticsearch/issues/118217 # Examples: # From 4a8a8a0bfe337aff50774c2c398060399a4d93e5 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sun, 8 Dec 2024 18:18:14 +1100 Subject: [PATCH 107/204] Mute org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT testEveryActionIsEitherOperatorOnlyOrNonOperator #118220 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 3977b63224f24..bfcac3a02b163 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -289,6 +289,9 @@ tests: - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT method: test {p0=data_stream/120_data_streams_stats/Multiple data stream} issue: https://github.com/elastic/elasticsearch/issues/118217 +- class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT + method: testEveryActionIsEitherOperatorOnlyOrNonOperator + issue: https://github.com/elastic/elasticsearch/issues/118220 # Examples: # From c09c4f427709f2a4373372f42f74abb556273c57 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Mon, 9 Dec 2024 08:27:23 +1100 Subject: [PATCH 108/204] Mute org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT #118224 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index bfcac3a02b163..a367234691895 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -292,6 +292,8 @@ tests: - class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT method: testEveryActionIsEitherOperatorOnlyOrNonOperator issue: https://github.com/elastic/elasticsearch/issues/118220 +- class: org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT + issue: https://github.com/elastic/elasticsearch/issues/118224 # Examples: # From 8107cc9e5e234447db9132068973c135fec2ff36 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Sun, 8 Dec 2024 17:56:09 -0600 Subject: [PATCH 109/204] Adding reindex data stream rest action (#118109) * Adding a _migration/reindex endpoint * Adding rest api spec and test * Adding a feature flag for reindex data streams * updating json spec * fixing a typo * Changing mode to an enum * Moving ParseFields into public static finals * Commenting out test that leaves task running, until we add a cancel API * Removing persistent task id from output * replacing a string with a variable --- .../rest-api-spec/api/migrate.reindex.json | 29 +++++ .../ReindexDataStreamTransportActionIT.java | 10 +- .../xpack/migrate/MigratePlugin.java | 102 +++++++++++++----- .../action/ReindexDataStreamAction.java | 89 +++++++++++++-- .../rest/RestMigrationReindexAction.java | 64 +++++++++++ .../action/ReindexDataStreamRequestTests.java | 39 +++++++ .../ReindexDataStreamResponseTests.java | 2 +- .../rest-api-spec/test/migrate/10_reindex.yml | 89 +++++++++++++++ 8 files changed, 387 insertions(+), 37 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/migrate.reindex.json create mode 100644 x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/rest/RestMigrationReindexAction.java create mode 100644 x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamRequestTests.java create mode 100644 x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/10_reindex.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.reindex.json b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.reindex.json new file mode 100644 index 0000000000000..149a90bc198b0 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.reindex.json @@ -0,0 +1,29 @@ +{ + "migrate.reindex":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-stream-reindex.html", + "description":"This API reindexes all legacy backing indices for a data stream. It does this in a persistent task. The persistent task id is returned immediately, and the reindexing work is completed in that task" + }, + "stability":"experimental", + "visibility":"private", + "headers":{ + "accept": [ "application/json"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_migration/reindex", + "methods":[ + "POST" + ] + } + ] + }, + "body":{ + "description":"The body contains the fields `mode` and `source.index, where the only mode currently supported is `upgrade`, and the `source.index` must be a data stream name", + "required":true + } + } +} + diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java index 3b68fc9995b57..62716e11f1720 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java @@ -51,7 +51,10 @@ protected Collection> nodePlugins() { public void testNonExistentDataStream() { String nonExistentDataStreamName = randomAlphaOfLength(50); - ReindexDataStreamRequest reindexDataStreamRequest = new ReindexDataStreamRequest(nonExistentDataStreamName); + ReindexDataStreamRequest reindexDataStreamRequest = new ReindexDataStreamRequest( + ReindexDataStreamAction.Mode.UPGRADE, + nonExistentDataStreamName + ); assertThrows( ResourceNotFoundException.class, () -> client().execute(new ActionType(ReindexDataStreamAction.NAME), reindexDataStreamRequest) @@ -61,7 +64,10 @@ public void testNonExistentDataStream() { public void testAlreadyUpToDateDataStream() throws Exception { String dataStreamName = randomAlphaOfLength(50).toLowerCase(Locale.ROOT); - ReindexDataStreamRequest reindexDataStreamRequest = new ReindexDataStreamRequest(dataStreamName); + ReindexDataStreamRequest reindexDataStreamRequest = new ReindexDataStreamRequest( + ReindexDataStreamAction.Mode.UPGRADE, + dataStreamName + ); createDataStream(dataStreamName); ReindexDataStreamResponse response = client().execute( new ActionType(ReindexDataStreamAction.NAME), diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java index 118cd69ece4d6..ac9e38da07421 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java @@ -11,21 +11,30 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.persistent.PersistentTaskState; import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction; import org.elasticsearch.xpack.migrate.action.ReindexDataStreamTransportAction; +import org.elasticsearch.xpack.migrate.rest.RestMigrationReindexAction; import org.elasticsearch.xpack.migrate.task.ReindexDataStreamPersistentTaskExecutor; import org.elasticsearch.xpack.migrate.task.ReindexDataStreamPersistentTaskState; import org.elasticsearch.xpack.migrate.task.ReindexDataStreamStatus; @@ -34,47 +43,80 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Predicate; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.REINDEX_DATA_STREAM_FEATURE_FLAG; public class MigratePlugin extends Plugin implements ActionPlugin, PersistentTaskPlugin { + @Override + public List getRestHandlers( + Settings unused, + NamedWriteableRegistry namedWriteableRegistry, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster, + Predicate clusterSupportsFeature + ) { + List handlers = new ArrayList<>(); + if (REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()) { + handlers.add(new RestMigrationReindexAction()); + } + return handlers; + } + @Override public List> getActions() { List> actions = new ArrayList<>(); - actions.add(new ActionHandler<>(ReindexDataStreamAction.INSTANCE, ReindexDataStreamTransportAction.class)); + if (REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()) { + actions.add(new ActionHandler<>(ReindexDataStreamAction.INSTANCE, ReindexDataStreamTransportAction.class)); + } return actions; } @Override public List getNamedXContent() { - return List.of( - new NamedXContentRegistry.Entry( - PersistentTaskState.class, - new ParseField(ReindexDataStreamPersistentTaskState.NAME), - ReindexDataStreamPersistentTaskState::fromXContent - ), - new NamedXContentRegistry.Entry( - PersistentTaskParams.class, - new ParseField(ReindexDataStreamTaskParams.NAME), - ReindexDataStreamTaskParams::fromXContent - ) - ); + if (REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()) { + return List.of( + new NamedXContentRegistry.Entry( + PersistentTaskState.class, + new ParseField(ReindexDataStreamPersistentTaskState.NAME), + ReindexDataStreamPersistentTaskState::fromXContent + ), + new NamedXContentRegistry.Entry( + PersistentTaskParams.class, + new ParseField(ReindexDataStreamTaskParams.NAME), + ReindexDataStreamTaskParams::fromXContent + ) + ); + } else { + return List.of(); + } } @Override public List getNamedWriteables() { - return List.of( - new NamedWriteableRegistry.Entry( - PersistentTaskState.class, - ReindexDataStreamPersistentTaskState.NAME, - ReindexDataStreamPersistentTaskState::new - ), - new NamedWriteableRegistry.Entry( - PersistentTaskParams.class, - ReindexDataStreamTaskParams.NAME, - ReindexDataStreamTaskParams::new - ), - new NamedWriteableRegistry.Entry(Task.Status.class, ReindexDataStreamStatus.NAME, ReindexDataStreamStatus::new) - ); + if (REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()) { + return List.of( + new NamedWriteableRegistry.Entry( + PersistentTaskState.class, + ReindexDataStreamPersistentTaskState.NAME, + ReindexDataStreamPersistentTaskState::new + ), + new NamedWriteableRegistry.Entry( + PersistentTaskParams.class, + ReindexDataStreamTaskParams.NAME, + ReindexDataStreamTaskParams::new + ), + new NamedWriteableRegistry.Entry(Task.Status.class, ReindexDataStreamStatus.NAME, ReindexDataStreamStatus::new) + ); + } else { + return List.of(); + } } @Override @@ -85,6 +127,12 @@ public List> getPersistentTasksExecutor( SettingsModule settingsModule, IndexNameExpressionResolver expressionResolver ) { - return List.of(new ReindexDataStreamPersistentTaskExecutor(client, clusterService, ReindexDataStreamTask.TASK_NAME, threadPool)); + if (REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()) { + return List.of( + new ReindexDataStreamPersistentTaskExecutor(client, clusterService, ReindexDataStreamTask.TASK_NAME, threadPool) + ); + } else { + return List.of(); + } } } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java index 1785e6971f824..eb7a910df8c0c 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java @@ -11,23 +11,41 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.FeatureFlag; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; +import java.util.Locale; import java.util.Objects; +import java.util.function.Predicate; public class ReindexDataStreamAction extends ActionType { + public static final FeatureFlag REINDEX_DATA_STREAM_FEATURE_FLAG = new FeatureFlag("reindex_data_stream"); public static final ReindexDataStreamAction INSTANCE = new ReindexDataStreamAction(); public static final String NAME = "indices:admin/data_stream/reindex"; + public static final ParseField MODE_FIELD = new ParseField("mode"); + public static final ParseField SOURCE_FIELD = new ParseField("source"); + public static final ParseField INDEX_FIELD = new ParseField("index"); public ReindexDataStreamAction() { super(NAME); } + public enum Mode { + UPGRADE + } + public static class ReindexDataStreamResponse extends ActionResponse implements ToXContentObject { private final String taskId; @@ -49,7 +67,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field("task", getTaskId()); + builder.field("acknowledged", true); builder.endObject(); return builder; } @@ -70,22 +88,52 @@ public boolean equals(Object other) { } - public static class ReindexDataStreamRequest extends ActionRequest { + public static class ReindexDataStreamRequest extends ActionRequest implements IndicesRequest, ToXContent { + private final Mode mode; private final String sourceDataStream; - public ReindexDataStreamRequest(String sourceDataStream) { - super(); + public ReindexDataStreamRequest(Mode mode, String sourceDataStream) { + this.mode = mode; this.sourceDataStream = sourceDataStream; } public ReindexDataStreamRequest(StreamInput in) throws IOException { super(in); + this.mode = Mode.valueOf(in.readString()); this.sourceDataStream = in.readString(); } + private static final ConstructingObjectParser> PARSER = + new ConstructingObjectParser<>("migration_reindex", objects -> { + Mode mode = Mode.valueOf(((String) objects[0]).toUpperCase(Locale.ROOT)); + String source = (String) objects[1]; + return new ReindexDataStreamRequest(mode, source); + }); + + private static final ConstructingObjectParser SOURCE_PARSER = new ConstructingObjectParser<>( + SOURCE_FIELD.getPreferredName(), + false, + (a, id) -> (String) a[0] + ); + + static { + SOURCE_PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), MODE_FIELD); + PARSER.declareObject( + ConstructingObjectParser.constructorArg(), + (parser, id) -> SOURCE_PARSER.apply(parser, null), + SOURCE_FIELD + ); + } + + public static ReindexDataStreamRequest fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); + out.writeString(mode.name()); out.writeString(sourceDataStream); } @@ -103,15 +151,42 @@ public String getSourceDataStream() { return sourceDataStream; } + public Mode getMode() { + return mode; + } + @Override public int hashCode() { - return Objects.hashCode(sourceDataStream); + return Objects.hash(mode, sourceDataStream); } @Override public boolean equals(Object other) { - return other instanceof ReindexDataStreamRequest - && sourceDataStream.equals(((ReindexDataStreamRequest) other).sourceDataStream); + return other instanceof ReindexDataStreamRequest otherRequest + && mode.equals(otherRequest.mode) + && sourceDataStream.equals(otherRequest.sourceDataStream); + } + + @Override + public String[] indices() { + return new String[] { sourceDataStream }; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + } + + /* + * This only exists for the sake of testing the xcontent parser + */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(MODE_FIELD.getPreferredName(), mode); + builder.startObject(SOURCE_FIELD.getPreferredName()); + builder.field(INDEX_FIELD.getPreferredName(), sourceDataStream); + builder.endObject(); + return builder; } } } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/rest/RestMigrationReindexAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/rest/RestMigrationReindexAction.java new file mode 100644 index 0000000000000..a7f630d68234d --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/rest/RestMigrationReindexAction.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.rest; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.RestBuilderListener; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction; +import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.ReindexDataStreamResponse; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.POST; + +public class RestMigrationReindexAction extends BaseRestHandler { + + @Override + public String getName() { + return "migration_reindex"; + } + + @Override + public List routes() { + return List.of(new Route(POST, "/_migration/reindex")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + ReindexDataStreamAction.ReindexDataStreamRequest reindexRequest; + try (XContentParser parser = request.contentParser()) { + reindexRequest = ReindexDataStreamAction.ReindexDataStreamRequest.fromXContent(parser); + } + return channel -> client.execute( + ReindexDataStreamAction.INSTANCE, + reindexRequest, + new ReindexDataStreamRestToXContentListener(channel) + ); + } + + static class ReindexDataStreamRestToXContentListener extends RestBuilderListener { + + ReindexDataStreamRestToXContentListener(RestChannel channel) { + super(channel); + } + + @Override + public RestResponse buildResponse(ReindexDataStreamResponse response, XContentBuilder builder) throws Exception { + response.toXContent(builder, channel.request()); + return new RestResponse(RestStatus.OK, builder); + } + } +} diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamRequestTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamRequestTests.java new file mode 100644 index 0000000000000..9c7bf87b6cff0 --- /dev/null +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamRequestTests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.action; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.ReindexDataStreamRequest; + +import java.io.IOException; + +public class ReindexDataStreamRequestTests extends AbstractXContentSerializingTestCase { + + @Override + protected ReindexDataStreamRequest createTestInstance() { + return new ReindexDataStreamRequest(ReindexDataStreamAction.Mode.UPGRADE, randomAlphaOfLength(40)); + } + + @Override + protected ReindexDataStreamRequest mutateInstance(ReindexDataStreamRequest instance) { + // There is currently only one possible value for mode, so we can't change it + return new ReindexDataStreamRequest(instance.getMode(), randomAlphaOfLength(50)); + } + + @Override + protected ReindexDataStreamRequest doParseInstance(XContentParser parser) throws IOException { + return ReindexDataStreamRequest.fromXContent(parser); + } + + @Override + protected Writeable.Reader instanceReader() { + return ReindexDataStreamRequest::new; + } +} diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamResponseTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamResponseTests.java index 06844577c4e36..d886fc660d7a8 100644 --- a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamResponseTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamResponseTests.java @@ -43,7 +43,7 @@ public void testToXContent() throws IOException { builder.humanReadable(true); response.toXContent(builder, EMPTY_PARAMS); try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - assertThat(parser.map(), equalTo(Map.of("task", response.getTaskId()))); + assertThat(parser.map(), equalTo(Map.of("acknowledged", true))); } } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/10_reindex.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/10_reindex.yml new file mode 100644 index 0000000000000..01a41b3aa8c94 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/10_reindex.yml @@ -0,0 +1,89 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + +--- +"Test Reindex With Unsupported Mode": + - do: + catch: /illegal_argument_exception/ + migrate.reindex: + body: | + { + "mode": "unsupported_mode", + "source": { + "index": "my-data-stream" + } + } + +--- +"Test Reindex With Nonexistent Data Stream": + - do: + catch: /resource_not_found_exception/ + migrate.reindex: + body: | + { + "mode": "upgrade", + "source": { + "index": "my-data-stream" + } + } + + - do: + catch: /resource_not_found_exception/ + migrate.reindex: + body: | + { + "mode": "upgrade", + "source": { + "index": "my-data-stream1,my-data-stream2" + } + } + + +--- +"Test Reindex With Bad Data Stream Name": + - do: + catch: /illegal_argument_exception/ + migrate.reindex: + body: | + { + "mode": "upgrade", + "source": { + "index": "my-data-stream*" + } + } + +--- +"Test Reindex With Existing Data Stream": + - do: + indices.put_index_template: + name: my-template1 + body: + index_patterns: [my-data-stream*] + template: + mappings: + properties: + '@timestamp': + type: date + 'foo': + type: keyword + data_stream: {} + + - do: + indices.create_data_stream: + name: my-data-stream + - is_true: acknowledged + +# Uncomment once the cancel API is in place +# - do: +# migrate.reindex: +# body: | +# { +# "mode": "upgrade", +# "source": { +# "index": "my-data-stream" +# } +# } +# - match: { task: "reindex-data-stream-my-data-stream" } From d411ad82daee10d298c2eb47d3fd900a83091baf Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Mon, 9 Dec 2024 13:44:10 +1100 Subject: [PATCH 110/204] AbstractRepositoryS3RestTestCase: Parse response for assertion (#118230) --- .../s3/AbstractRepositoryS3RestTestCase.java | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3RestTestCase.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3RestTestCase.java index 2199a64521759..67ada622efeea 100644 --- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3RestTestCase.java +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3RestTestCase.java @@ -19,6 +19,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ObjectPath; import java.io.Closeable; import java.io.IOException; @@ -27,7 +28,6 @@ import java.util.function.UnaryOperator; import java.util.stream.Collectors; -import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -152,10 +152,9 @@ private void testNonexistentBucket(Boolean readonly) throws Exception { final var responseException = expectThrows(ResponseException.class, () -> client().performRequest(registerRequest)); assertEquals(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), responseException.getResponse().getStatusLine().getStatusCode()); - assertThat( - responseException.getMessage(), - allOf(containsString("repository_verification_exception"), containsString("is not accessible on master node")) - ); + final var responseObjectPath = ObjectPath.createFromResponse(responseException.getResponse()); + assertThat(responseObjectPath.evaluate("error.type"), equalTo("repository_verification_exception")); + assertThat(responseObjectPath.evaluate("error.reason"), containsString("is not accessible on master node")); } public void testNonexistentClient() throws Exception { @@ -181,15 +180,11 @@ private void testNonexistentClient(Boolean readonly) throws Exception { final var responseException = expectThrows(ResponseException.class, () -> client().performRequest(registerRequest)); assertEquals(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), responseException.getResponse().getStatusLine().getStatusCode()); - assertThat( - responseException.getMessage(), - allOf( - containsString("repository_verification_exception"), - containsString("is not accessible on master node"), - containsString("illegal_argument_exception"), - containsString("Unknown s3 client name") - ) - ); + final var responseObjectPath = ObjectPath.createFromResponse(responseException.getResponse()); + assertThat(responseObjectPath.evaluate("error.type"), equalTo("repository_verification_exception")); + assertThat(responseObjectPath.evaluate("error.reason"), containsString("is not accessible on master node")); + assertThat(responseObjectPath.evaluate("error.caused_by.type"), equalTo("illegal_argument_exception")); + assertThat(responseObjectPath.evaluate("error.caused_by.reason"), containsString("Unknown s3 client name")); } public void testNonexistentSnapshot() throws Exception { @@ -212,7 +207,8 @@ private void testNonexistentSnapshot(Boolean readonly) throws Exception { final var getSnapshotRequest = new Request("GET", "/_snapshot/" + repositoryName + "/" + randomIdentifier()); final var getSnapshotException = expectThrows(ResponseException.class, () -> client().performRequest(getSnapshotRequest)); assertEquals(RestStatus.NOT_FOUND.getStatus(), getSnapshotException.getResponse().getStatusLine().getStatusCode()); - assertThat(getSnapshotException.getMessage(), containsString("snapshot_missing_exception")); + final var getResponseObjectPath = ObjectPath.createFromResponse(getSnapshotException.getResponse()); + assertThat(getResponseObjectPath.evaluate("error.type"), equalTo("snapshot_missing_exception")); final var restoreRequest = new Request("POST", "/_snapshot/" + repositoryName + "/" + randomIdentifier() + "/_restore"); if (randomBoolean()) { @@ -220,13 +216,15 @@ private void testNonexistentSnapshot(Boolean readonly) throws Exception { } final var restoreException = expectThrows(ResponseException.class, () -> client().performRequest(restoreRequest)); assertEquals(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), restoreException.getResponse().getStatusLine().getStatusCode()); - assertThat(restoreException.getMessage(), containsString("snapshot_restore_exception")); + final var restoreResponseObjectPath = ObjectPath.createFromResponse(restoreException.getResponse()); + assertThat(restoreResponseObjectPath.evaluate("error.type"), equalTo("snapshot_restore_exception")); if (readonly != Boolean.TRUE) { final var deleteRequest = new Request("DELETE", "/_snapshot/" + repositoryName + "/" + randomIdentifier()); final var deleteException = expectThrows(ResponseException.class, () -> client().performRequest(deleteRequest)); assertEquals(RestStatus.NOT_FOUND.getStatus(), deleteException.getResponse().getStatusLine().getStatusCode()); - assertThat(deleteException.getMessage(), containsString("snapshot_missing_exception")); + final var deleteResponseObjectPath = ObjectPath.createFromResponse(deleteException.getResponse()); + assertThat(deleteResponseObjectPath.evaluate("error.type"), equalTo("snapshot_missing_exception")); } } } From d7a9c50cf24abe11c76a14da4c6e304480eea728 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:32:09 +1100 Subject: [PATCH 111/204] Mute org.elasticsearch.packaging.test.ArchiveTests test60StartAndStop #118216 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index a367234691895..4523db7239be6 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -294,6 +294,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/118220 - class: org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT issue: https://github.com/elastic/elasticsearch/issues/118224 +- class: org.elasticsearch.packaging.test.ArchiveTests + method: test60StartAndStop + issue: https://github.com/elastic/elasticsearch/issues/118216 # Examples: # From b2b8e3f762753ac903f056a4238a204e993bb8d5 Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:05:11 +0100 Subject: [PATCH 112/204] [DOCS] [8.17] Adds new default inference endpoint information (#117985) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adds new default inference information * Update docs/reference/mapping/types/semantic-text.asciidoc Co-authored-by: István Zoltán Szabó * Update docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc Co-authored-by: István Zoltán Szabó * Update docs/reference/mapping/types/semantic-text.asciidoc Co-authored-by: David Kyle --------- Co-authored-by: István Zoltán Szabó Co-authored-by: David Kyle --- .../mapping/types/semantic-text.asciidoc | 19 +++---- .../semantic-search-semantic-text.asciidoc | 8 +-- .../semantic-text-hybrid-search | 51 +++---------------- 3 files changed, 21 insertions(+), 57 deletions(-) diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc index b3e103ec6dbd9..96dc402e10c60 100644 --- a/docs/reference/mapping/types/semantic-text.asciidoc +++ b/docs/reference/mapping/types/semantic-text.asciidoc @@ -12,13 +12,14 @@ Long passages are <> to smaller secti The `semantic_text` field type specifies an inference endpoint identifier that will be used to generate embeddings. You can create the inference endpoint by using the <>. -This field type and the <> type make it simpler to perform semantic search on your data. -If you don't specify an inference endpoint, the <> is used by default. +This field type and the <> type make it simpler to perform semantic search on your data. + +If you don’t specify an inference endpoint, the `inference_id` field defaults to `.elser-2-elasticsearch`, a preconfigured endpoint for the elasticsearch service. Using `semantic_text`, you won't need to specify how to generate embeddings for your data, or how to index it. The {infer} endpoint automatically determines the embedding generation, indexing, and query to use. -If you use the ELSER service, you can set up `semantic_text` with the following API request: +If you use the preconfigured `.elser-2-elasticsearch` endpoint, you can set up `semantic_text` with the following API request: [source,console] ------------------------------------------------------------ @@ -34,7 +35,7 @@ PUT my-index-000001 } ------------------------------------------------------------ -If you use a service other than ELSER, you must create an {infer} endpoint using the <> and reference it when setting up `semantic_text` as the following example demonstrates: +To use a custom {infer} endpoint instead of the default `.elser-2-elasticsearch`, you must <> and specify its `inference_id` when setting up the `semantic_text` field type. [source,console] ------------------------------------------------------------ @@ -53,8 +54,7 @@ PUT my-index-000002 // TEST[skip:Requires inference endpoint] <1> The `inference_id` of the {infer} endpoint to use to generate embeddings. - -The recommended way to use semantic_text is by having dedicated {infer} endpoints for ingestion and search. +The recommended way to use `semantic_text` is by having dedicated {infer} endpoints for ingestion and search. This ensures that search speed remains unaffected by ingestion workloads, and vice versa. After creating dedicated {infer} endpoints for both, you can reference them using the `inference_id` and `search_inference_id` parameters when setting up the index mapping for an index that uses the `semantic_text` field. @@ -82,10 +82,11 @@ PUT my-index-000003 `inference_id`:: (Required, string) -{infer-cap} endpoint that will be used to generate the embeddings for the field. +{infer-cap} endpoint that will be used to generate embeddings for the field. +By default, `.elser-2-elasticsearch` is used. This parameter cannot be updated. Use the <> to create the endpoint. -If `search_inference_id` is specified, the {infer} endpoint defined by `inference_id` will only be used at index time. +If `search_inference_id` is specified, the {infer} endpoint will only be used at index time. `search_inference_id`:: (Optional, string) @@ -201,7 +202,7 @@ PUT test-index "properties": { "infer_field": { "type": "semantic_text", - "inference_id": "my-elser-endpoint" + "inference_id": ".elser-2-elasticsearch" }, "source_field": { "type": "text", diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc index ba9c81db21384..3448940b6fad7 100644 --- a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc @@ -14,15 +14,15 @@ You don't need to define model related settings and parameters, or create {infer The recommended way to use <> in the {stack} is following the `semantic_text` workflow. When you need more control over indexing and query settings, you can still use the complete {infer} workflow (refer to <> to review the process). -This tutorial uses the <> for demonstration, but you can use any service and their supported models offered by the {infer-cap} API. +This tutorial uses the <> for demonstration, but you can use any service and their supported models offered by the {infer-cap} API. [discrete] [[semantic-text-requirements]] ==== Requirements -This tutorial uses the <> for demonstration, which is created automatically as needed. -To use the `semantic_text` field type with an {infer} service other than ELSER, you must create an inference endpoint using the <>. +This tutorial uses the <> for demonstration, which is created automatically as needed. +To use the `semantic_text` field type with an {infer} service other than `elasticsearch` service, you must create an inference endpoint using the <>. [discrete] @@ -48,7 +48,7 @@ PUT semantic-embeddings // TEST[skip:TBD] <1> The name of the field to contain the generated embeddings. <2> The field to contain the embeddings is a `semantic_text` field. -Since no `inference_id` is provided, the <> is used by default. +Since no `inference_id` is provided, the default endpoint `.elser-2-elasticsearch` for the <> is used. To use a different {infer} service, you must create an {infer} endpoint first using the <> and then specify it in the `semantic_text` field mapping using the `inference_id` parameter. [NOTE] diff --git a/docs/reference/search/search-your-data/semantic-text-hybrid-search b/docs/reference/search/search-your-data/semantic-text-hybrid-search index c56b283434df5..4b49a7c3155db 100644 --- a/docs/reference/search/search-your-data/semantic-text-hybrid-search +++ b/docs/reference/search/search-your-data/semantic-text-hybrid-search @@ -8,47 +8,12 @@ This tutorial demonstrates how to perform hybrid search, combining semantic sear In hybrid search, semantic search retrieves results based on the meaning of the text, while full-text search focuses on exact word matches. By combining both methods, hybrid search delivers more relevant results, particularly in cases where relying on a single approach may not be sufficient. -The recommended way to use hybrid search in the {stack} is following the `semantic_text` workflow. This tutorial uses the <> for demonstration, but you can use any service and its supported models offered by the {infer-cap} API. - -[discrete] -[[semantic-text-hybrid-infer-endpoint]] -==== Create the {infer} endpoint - -Create an inference endpoint by using the <>: - -[source,console] ------------------------------------------------------------- -PUT _inference/sparse_embedding/my-elser-endpoint <1> -{ - "service": "elser", <2> - "service_settings": { - "adaptive_allocations": { <3> - "enabled": true, - "min_number_of_allocations": 3, - "max_number_of_allocations": 10 - }, - "num_threads": 1 - } -} ------------------------------------------------------------- -// TEST[skip:TBD] -<1> The task type is `sparse_embedding` in the path as the `elser` service will -be used and ELSER creates sparse vectors. The `inference_id` is -`my-elser-endpoint`. -<2> The `elser` service is used in this example. -<3> This setting enables and configures adaptive allocations. -Adaptive allocations make it possible for ELSER to automatically scale up or down resources based on the current load on the process. - -[NOTE] -==== -You might see a 502 bad gateway error in the response when using the {kib} Console. -This error usually just reflects a timeout, while the model downloads in the background. -You can check the download progress in the {ml-app} UI. -==== +The recommended way to use hybrid search in the {stack} is following the `semantic_text` workflow. +This tutorial uses the <> for demonstration, but you can use any service and their supported models offered by the {infer-cap} API. [discrete] [[hybrid-search-create-index-mapping]] -==== Create an index mapping for hybrid search +==== Create an index mapping The destination index will contain both the embeddings for semantic search and the original text field for full-text search. This structure enables the combination of semantic search and full-text search. @@ -60,11 +25,10 @@ PUT semantic-embeddings "properties": { "semantic_text": { <1> "type": "semantic_text", - "inference_id": "my-elser-endpoint" <2> }, - "content": { <3> + "content": { <2> "type": "text", - "copy_to": "semantic_text" <4> + "copy_to": "semantic_text" <3> } } } @@ -72,9 +36,8 @@ PUT semantic-embeddings ------------------------------------------------------------ // TEST[skip:TBD] <1> The name of the field to contain the generated embeddings for semantic search. -<2> The identifier of the inference endpoint that generates the embeddings based on the input text. -<3> The name of the field to contain the original text for lexical search. -<4> The textual data stored in the `content` field will be copied to `semantic_text` and processed by the {infer} endpoint. +<2> The name of the field to contain the original text for lexical search. +<3> The textual data stored in the `content` field will be copied to `semantic_text` and processed by the {infer} endpoint. [NOTE] ==== From 7ffac3b3f3da18e46af023341f912b83899f0863 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 9 Dec 2024 09:08:01 +0100 Subject: [PATCH 113/204] Save O(1s) of CPU time in FieldSortIT (#118146) I see this `toString` take ~2s of hot CPU time in some test runs which isn't entirely surprising. Rather than optimize this in some form, just dropping the string here which aligns the thing with other tests anyway. --- .../java/org/elasticsearch/search/sort/FieldSortIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java index 87665c3d784f1..bf7a315040caa 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -202,7 +202,6 @@ public void testIssue6614() throws InterruptedException { response -> { for (int j = 0; j < response.getHits().getHits().length; j++) { assertThat( - response.toString() + "\n vs. \n" + allDocsResponse.toString(), response.getHits().getHits()[j].getId(), equalTo(allDocsResponse.getHits().getHits()[j].getId()) ); From 22e8f61db903473b462bff9a8919b399d742162d Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 9 Dec 2024 10:36:27 +0100 Subject: [PATCH 114/204] Unmute test around can match shards skipping against searchable snapshots (#118189) This test has been muted for a long time. The failure may or may no longer be relevant. This commit unmutes it. If it fails again, we'll get updated info and look into it. Closes #105339 --- .../SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java index d4bbd4495df26..23e414c0dc1bf 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java @@ -42,7 +42,6 @@ import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.NodeRoles; -import org.elasticsearch.test.junit.annotations.TestIssueLogging; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction; @@ -788,11 +787,6 @@ public void testQueryPhaseIsExecutedInAnAvailableNodeWhenAllShardsCanBeSkipped() * Can match against searchable snapshots is tested via both the Search API and the SearchShards (transport-only) API. * The latter is a way to do only a can-match rather than all search phases. */ - @TestIssueLogging( - issueUrl = "https://github.com/elastic/elasticsearch/issues/97878", - value = "org.elasticsearch.snapshots:DEBUG,org.elasticsearch.indices.recovery:DEBUG,org.elasticsearch.action.search:DEBUG" - ) - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105339") public void testSearchableSnapshotShardsThatHaveMatchingDataAreNotSkippedOnTheCoordinatingNode() throws Exception { internalCluster().startMasterOnlyNode(); internalCluster().startCoordinatingOnlyNode(Settings.EMPTY); From 638e5b6de2daa785ba07548feb907ad9079a6c94 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 9 Dec 2024 09:59:53 +0000 Subject: [PATCH 115/204] Revert "Adding default endpoint for Elastic Rerank (#117939)" (#118221) This reverts commit 54c320ebc9b262e66ab92af660a8a155311059d4. --- docs/changelog/117939.yaml | 5 -- .../xpack/inference/DefaultEndPointsIT.java | 40 -------------- .../inference/InferenceBaseRestTest.java | 47 ++++------------ .../xpack/inference/InferenceCrudIT.java | 4 +- .../InferenceNamedWriteablesProvider.java | 6 +- .../elasticsearch/CustomElandRerankModel.java | 4 +- ...ava => CustomElandRerankTaskSettings.java} | 25 +++++---- .../elasticsearch/ElasticRerankerModel.java | 5 +- .../ElasticsearchInternalService.java | 55 ++++++------------- ...> CustomElandRerankTaskSettingsTests.java} | 48 ++++++++-------- .../ElasticsearchInternalServiceTests.java | 42 +++++++------- 11 files changed, 96 insertions(+), 185 deletions(-) delete mode 100644 docs/changelog/117939.yaml rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/{RerankTaskSettings.java => CustomElandRerankTaskSettings.java} (79%) rename x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/{RerankTaskSettingsTests.java => CustomElandRerankTaskSettingsTests.java} (53%) diff --git a/docs/changelog/117939.yaml b/docs/changelog/117939.yaml deleted file mode 100644 index d41111f099f97..0000000000000 --- a/docs/changelog/117939.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117939 -summary: Adding default endpoint for Elastic Rerank -area: Machine Learning -type: enhancement -issues: [] diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java index 068b3e1f4ce04..ba3e48e11928d 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java @@ -57,9 +57,6 @@ public void testGet() throws IOException { var e5Model = getModel(ElasticsearchInternalService.DEFAULT_E5_ID); assertDefaultE5Config(e5Model); - - var rerankModel = getModel(ElasticsearchInternalService.DEFAULT_RERANK_ID); - assertDefaultRerankConfig(rerankModel); } @SuppressWarnings("unchecked") @@ -128,42 +125,6 @@ private static void assertDefaultE5Config(Map modelConfig) { assertDefaultChunkingSettings(modelConfig); } - @SuppressWarnings("unchecked") - public void testInferDeploysDefaultRerank() throws IOException { - var model = getModel(ElasticsearchInternalService.DEFAULT_RERANK_ID); - assertDefaultRerankConfig(model); - - var inputs = List.of("Hello World", "Goodnight moon"); - var query = "but why"; - var queryParams = Map.of("timeout", "120s"); - var results = infer(ElasticsearchInternalService.DEFAULT_RERANK_ID, TaskType.RERANK, inputs, query, queryParams); - var embeddings = (List>) results.get("rerank"); - assertThat(results.toString(), embeddings, hasSize(2)); - } - - @SuppressWarnings("unchecked") - private static void assertDefaultRerankConfig(Map modelConfig) { - assertEquals(modelConfig.toString(), ElasticsearchInternalService.DEFAULT_RERANK_ID, modelConfig.get("inference_id")); - assertEquals(modelConfig.toString(), ElasticsearchInternalService.NAME, modelConfig.get("service")); - assertEquals(modelConfig.toString(), TaskType.RERANK.toString(), modelConfig.get("task_type")); - - var serviceSettings = (Map) modelConfig.get("service_settings"); - assertThat(modelConfig.toString(), serviceSettings.get("model_id"), is(".rerank-v1")); - assertEquals(modelConfig.toString(), 1, serviceSettings.get("num_threads")); - - var adaptiveAllocations = (Map) serviceSettings.get("adaptive_allocations"); - assertThat( - modelConfig.toString(), - adaptiveAllocations, - Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 32)) - ); - - var chunkingSettings = (Map) modelConfig.get("chunking_settings"); - assertNull(chunkingSettings); - var taskSettings = (Map) modelConfig.get("task_settings"); - assertThat(modelConfig.toString(), taskSettings, Matchers.is(Map.of("return_documents", true))); - } - @SuppressWarnings("unchecked") private static void assertDefaultChunkingSettings(Map modelConfig) { var chunkingSettings = (Map) modelConfig.get("chunking_settings"); @@ -198,7 +159,6 @@ public void onFailure(Exception exception) { var request = createInferenceRequest( Strings.format("_inference/%s", ElasticsearchInternalService.DEFAULT_ELSER_ID), inputs, - null, queryParams ); client().performRequestAsync(request, listener); diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java index 1716057cdfe46..07ce2fe00642b 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java @@ -336,7 +336,7 @@ private List getInternalAsList(String endpoint) throws IOException { protected Map infer(String modelId, List input) throws IOException { var endpoint = Strings.format("_inference/%s", modelId); - return inferInternal(endpoint, input, null, Map.of()); + return inferInternal(endpoint, input, Map.of()); } protected Deque streamInferOnMockService(String modelId, TaskType taskType, List input) throws Exception { @@ -352,7 +352,7 @@ protected Deque unifiedCompletionInferOnMockService(String mode private Deque callAsync(String endpoint, List input) throws Exception { var request = new Request("POST", endpoint); - request.setJsonEntity(jsonBody(input, null)); + request.setJsonEntity(jsonBody(input)); return execAsyncCall(request); } @@ -394,60 +394,33 @@ private String createUnifiedJsonBody(List input, String role) throws IOE protected Map infer(String modelId, TaskType taskType, List input) throws IOException { var endpoint = Strings.format("_inference/%s/%s", taskType, modelId); - return inferInternal(endpoint, input, null, Map.of()); + return inferInternal(endpoint, input, Map.of()); } protected Map infer(String modelId, TaskType taskType, List input, Map queryParameters) throws IOException { var endpoint = Strings.format("_inference/%s/%s?error_trace", taskType, modelId); - return inferInternal(endpoint, input, null, queryParameters); + return inferInternal(endpoint, input, queryParameters); } - protected Map infer( - String modelId, - TaskType taskType, - List input, - String query, - Map queryParameters - ) throws IOException { - var endpoint = Strings.format("_inference/%s/%s?error_trace", taskType, modelId); - return inferInternal(endpoint, input, query, queryParameters); - } - - protected Request createInferenceRequest( - String endpoint, - List input, - @Nullable String query, - Map queryParameters - ) { + protected Request createInferenceRequest(String endpoint, List input, Map queryParameters) { var request = new Request("POST", endpoint); - request.setJsonEntity(jsonBody(input, query)); + request.setJsonEntity(jsonBody(input)); if (queryParameters.isEmpty() == false) { request.addParameters(queryParameters); } return request; } - private Map inferInternal( - String endpoint, - List input, - @Nullable String query, - Map queryParameters - ) throws IOException { - var request = createInferenceRequest(endpoint, input, query, queryParameters); + private Map inferInternal(String endpoint, List input, Map queryParameters) throws IOException { + var request = createInferenceRequest(endpoint, input, queryParameters); var response = client().performRequest(request); assertOkOrCreated(response); return entityAsMap(response); } - private String jsonBody(List input, @Nullable String query) { - final StringBuilder bodyBuilder = new StringBuilder("{"); - - if (query != null) { - bodyBuilder.append("\"query\":\"").append(query).append("\","); - } - - bodyBuilder.append("\"input\": ["); + private String jsonBody(List input) { + var bodyBuilder = new StringBuilder("{\"input\": ["); for (var in : input) { bodyBuilder.append('"').append(in).append('"').append(','); } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index 2099ec8287a76..1e19491aeaa60 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -49,7 +49,7 @@ public void testCRUD() throws IOException { } var getAllModels = getAllModels(); - int numModels = 12; + int numModels = 11; assertThat(getAllModels, hasSize(numModels)); var getSparseModels = getModels("_all", TaskType.SPARSE_EMBEDDING); @@ -537,7 +537,7 @@ private static String expectedResult(String input) { } public void testGetZeroModels() throws IOException { - var models = getModels("_all", TaskType.COMPLETION); + var models = getModels("_all", TaskType.RERANK); assertThat(models, empty()); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java index a4187f4c4fa90..b83c098ca808c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java @@ -63,12 +63,12 @@ import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalTextEmbeddingServiceSettings; +import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandRerankTaskSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticRerankerServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElserInternalServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElserMlNodeTaskSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.MultilingualE5SmallInternalServiceSettings; -import org.elasticsearch.xpack.inference.services.elasticsearch.RerankTaskSettings; import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionServiceSettings; import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.googlevertexai.GoogleVertexAiSecretSettings; @@ -518,7 +518,9 @@ private static void addCustomElandWriteables(final List namedWriteables) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java index 6388bb33bb78d..f620b15680c8d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java @@ -17,7 +17,7 @@ import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.elasticsearch.RerankTaskSettings.RETURN_DOCUMENTS; +import static org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandRerankTaskSettings.RETURN_DOCUMENTS; public class CustomElandRerankModel extends CustomElandModel { @@ -26,7 +26,7 @@ public CustomElandRerankModel( TaskType taskType, String service, CustomElandInternalServiceSettings serviceSettings, - RerankTaskSettings taskSettings + CustomElandRerankTaskSettings taskSettings ) { super(inferenceEntityId, taskType, service, serviceSettings, taskSettings); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/RerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java similarity index 79% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/RerankTaskSettings.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java index 3c25f7a6a9016..a0be1661b860d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/RerankTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java @@ -26,14 +26,14 @@ /** * Defines the task settings for internal rerank service. */ -public class RerankTaskSettings implements TaskSettings { +public class CustomElandRerankTaskSettings implements TaskSettings { public static final String NAME = "custom_eland_rerank_task_settings"; public static final String RETURN_DOCUMENTS = "return_documents"; - static final RerankTaskSettings DEFAULT_SETTINGS = new RerankTaskSettings(Boolean.TRUE); + static final CustomElandRerankTaskSettings DEFAULT_SETTINGS = new CustomElandRerankTaskSettings(Boolean.TRUE); - public static RerankTaskSettings defaultsFromMap(Map map) { + public static CustomElandRerankTaskSettings defaultsFromMap(Map map) { ValidationException validationException = new ValidationException(); if (map == null || map.isEmpty()) { @@ -49,7 +49,7 @@ public static RerankTaskSettings defaultsFromMap(Map map) { returnDocuments = true; } - return new RerankTaskSettings(returnDocuments); + return new CustomElandRerankTaskSettings(returnDocuments); } /** @@ -57,13 +57,13 @@ public static RerankTaskSettings defaultsFromMap(Map map) { * @param map source map * @return Task settings */ - public static RerankTaskSettings fromMap(Map map) { + public static CustomElandRerankTaskSettings fromMap(Map map) { if (map == null || map.isEmpty()) { return DEFAULT_SETTINGS; } Boolean returnDocuments = extractOptionalBoolean(map, RETURN_DOCUMENTS, new ValidationException()); - return new RerankTaskSettings(returnDocuments); + return new CustomElandRerankTaskSettings(returnDocuments); } /** @@ -74,17 +74,20 @@ public static RerankTaskSettings fromMap(Map map) { * @param requestTaskSettings the settings passed in within the task_settings field of the request * @return Either {@code originalSettings} or {@code requestTaskSettings} */ - public static RerankTaskSettings of(RerankTaskSettings originalSettings, RerankTaskSettings requestTaskSettings) { + public static CustomElandRerankTaskSettings of( + CustomElandRerankTaskSettings originalSettings, + CustomElandRerankTaskSettings requestTaskSettings + ) { return requestTaskSettings.returnDocuments() != null ? requestTaskSettings : originalSettings; } private final Boolean returnDocuments; - public RerankTaskSettings(StreamInput in) throws IOException { + public CustomElandRerankTaskSettings(StreamInput in) throws IOException { this(in.readOptionalBoolean()); } - public RerankTaskSettings(@Nullable Boolean doReturnDocuments) { + public CustomElandRerankTaskSettings(@Nullable Boolean doReturnDocuments) { if (doReturnDocuments == null) { this.returnDocuments = true; } else { @@ -130,7 +133,7 @@ public Boolean returnDocuments() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - RerankTaskSettings that = (RerankTaskSettings) o; + CustomElandRerankTaskSettings that = (CustomElandRerankTaskSettings) o; return Objects.equals(returnDocuments, that.returnDocuments); } @@ -141,7 +144,7 @@ public int hashCode() { @Override public TaskSettings updatedTaskSettings(Map newSettings) { - RerankTaskSettings updatedSettings = RerankTaskSettings.fromMap(new HashMap<>(newSettings)); + CustomElandRerankTaskSettings updatedSettings = CustomElandRerankTaskSettings.fromMap(new HashMap<>(newSettings)); return of(this, updatedSettings); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticRerankerModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticRerankerModel.java index 276bce6dbe8f8..115cc9f05599a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticRerankerModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticRerankerModel.java @@ -9,6 +9,7 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; @@ -21,9 +22,9 @@ public ElasticRerankerModel( TaskType taskType, String service, ElasticRerankerServiceSettings serviceSettings, - RerankTaskSettings taskSettings + ChunkingSettings chunkingSettings ) { - super(inferenceEntityId, taskType, service, serviceSettings, taskSettings); + super(inferenceEntityId, taskType, service, serviceSettings, chunkingSettings); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index 5f613d6be5869..8cb91782e238e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -103,7 +103,6 @@ public class ElasticsearchInternalService extends BaseElasticsearchInternalServi public static final int EMBEDDING_MAX_BATCH_SIZE = 10; public static final String DEFAULT_ELSER_ID = ".elser-2-elasticsearch"; public static final String DEFAULT_E5_ID = ".multilingual-e5-small-elasticsearch"; - public static final String DEFAULT_RERANK_ID = ".rerank-v1-elasticsearch"; private static final EnumSet supportedTaskTypes = EnumSet.of( TaskType.RERANK, @@ -228,7 +227,7 @@ public void parseRequestConfig( ) ); } else if (RERANKER_ID.equals(modelId)) { - rerankerCase(inferenceEntityId, taskType, config, serviceSettingsMap, taskSettingsMap, modelListener); + rerankerCase(inferenceEntityId, taskType, config, serviceSettingsMap, chunkingSettings, modelListener); } else { customElandCase(inferenceEntityId, taskType, serviceSettingsMap, taskSettingsMap, chunkingSettings, modelListener); } @@ -311,7 +310,7 @@ private static CustomElandModel createCustomElandModel( taskType, NAME, elandServiceSettings(serviceSettings, context), - RerankTaskSettings.fromMap(taskSettings) + CustomElandRerankTaskSettings.fromMap(taskSettings) ); default -> throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST); }; @@ -334,7 +333,7 @@ private void rerankerCase( TaskType taskType, Map config, Map serviceSettingsMap, - Map taskSettingsMap, + ChunkingSettings chunkingSettings, ActionListener modelListener ) { @@ -349,7 +348,7 @@ private void rerankerCase( taskType, NAME, new ElasticRerankerServiceSettings(esServiceSettingsBuilder.build()), - RerankTaskSettings.fromMap(taskSettingsMap) + chunkingSettings ) ); } @@ -515,14 +514,6 @@ public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, M ElserMlNodeTaskSettings.DEFAULT, chunkingSettings ); - } else if (modelId.equals(RERANKER_ID)) { - return new ElasticRerankerModel( - inferenceEntityId, - taskType, - NAME, - new ElasticRerankerServiceSettings(ElasticsearchInternalServiceSettings.fromPersistedMap(serviceSettingsMap)), - RerankTaskSettings.fromMap(taskSettingsMap) - ); } else { return createCustomElandModel( inferenceEntityId, @@ -674,23 +665,21 @@ public void inferRerank( ) { var request = buildInferenceRequest(model.mlNodeDeploymentId(), new TextSimilarityConfigUpdate(query), inputs, inputType, timeout); - var returnDocs = Boolean.TRUE; - if (model.getTaskSettings() instanceof RerankTaskSettings modelSettings) { - var requestSettings = RerankTaskSettings.fromMap(requestTaskSettings); - returnDocs = RerankTaskSettings.of(modelSettings, requestSettings).returnDocuments(); - } + var modelSettings = (CustomElandRerankTaskSettings) model.getTaskSettings(); + var requestSettings = CustomElandRerankTaskSettings.fromMap(requestTaskSettings); + Boolean returnDocs = CustomElandRerankTaskSettings.of(modelSettings, requestSettings).returnDocuments(); Function inputSupplier = returnDocs == Boolean.TRUE ? inputs::get : i -> null; - ActionListener mlResultsListener = listener.delegateFailureAndWrap( - (l, inferenceResult) -> l.onResponse(textSimilarityResultsToRankedDocs(inferenceResult.getInferenceResults(), inputSupplier)) - ); - - var maybeDeployListener = mlResultsListener.delegateResponse( - (l, exception) -> maybeStartDeployment(model, exception, request, mlResultsListener) + client.execute( + InferModelAction.INSTANCE, + request, + listener.delegateFailureAndWrap( + (l, inferenceResult) -> l.onResponse( + textSimilarityResultsToRankedDocs(inferenceResult.getInferenceResults(), inputSupplier) + ) + ) ); - - client.execute(InferModelAction.INSTANCE, request, maybeDeployListener); } public void chunkedInfer( @@ -834,8 +823,7 @@ private RankedDocsResults textSimilarityResultsToRankedDocs( public List defaultConfigIds() { return List.of( new DefaultConfigId(DEFAULT_ELSER_ID, TaskType.SPARSE_EMBEDDING, this), - new DefaultConfigId(DEFAULT_E5_ID, TaskType.TEXT_EMBEDDING, this), - new DefaultConfigId(DEFAULT_RERANK_ID, TaskType.RERANK, this) + new DefaultConfigId(DEFAULT_E5_ID, TaskType.TEXT_EMBEDDING, this) ); } @@ -928,19 +916,12 @@ private List defaultConfigs(boolean useLinuxOptimizedModel) { ), ChunkingSettingsBuilder.DEFAULT_SETTINGS ); - var defaultRerank = new ElasticRerankerModel( - DEFAULT_RERANK_ID, - TaskType.RERANK, - NAME, - new ElasticRerankerServiceSettings(null, 1, RERANKER_ID, new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 32)), - RerankTaskSettings.DEFAULT_SETTINGS - ); - return List.of(defaultElser, defaultE5, defaultRerank); + return List.of(defaultElser, defaultE5); } @Override boolean isDefaultId(String inferenceId) { - return DEFAULT_ELSER_ID.equals(inferenceId) || DEFAULT_E5_ID.equals(inferenceId) || DEFAULT_RERANK_ID.equals(inferenceId); + return DEFAULT_ELSER_ID.equals(inferenceId) || DEFAULT_E5_ID.equals(inferenceId); } static EmbeddingRequestChunker.EmbeddingType embeddingTypeFromTaskTypeAndSettings( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/RerankTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java similarity index 53% rename from x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/RerankTaskSettingsTests.java rename to x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java index 255454a1ed62b..4207896fc54f3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/RerankTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java @@ -22,7 +22,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.sameInstance; -public class RerankTaskSettingsTests extends AbstractWireSerializingTestCase { +public class CustomElandRerankTaskSettingsTests extends AbstractWireSerializingTestCase { public void testIsEmpty() { var randomSettings = createRandom(); @@ -35,9 +35,9 @@ public void testUpdatedTaskSettings() { var newSettings = createRandom(); Map newSettingsMap = new HashMap<>(); if (newSettings.returnDocuments() != null) { - newSettingsMap.put(RerankTaskSettings.RETURN_DOCUMENTS, newSettings.returnDocuments()); + newSettingsMap.put(CustomElandRerankTaskSettings.RETURN_DOCUMENTS, newSettings.returnDocuments()); } - RerankTaskSettings updatedSettings = (RerankTaskSettings) initialSettings.updatedTaskSettings( + CustomElandRerankTaskSettings updatedSettings = (CustomElandRerankTaskSettings) initialSettings.updatedTaskSettings( Collections.unmodifiableMap(newSettingsMap) ); if (newSettings.returnDocuments() == null) { @@ -48,37 +48,37 @@ public void testUpdatedTaskSettings() { } public void testDefaultsFromMap_MapIsNull_ReturnsDefaultSettings() { - var rerankTaskSettings = RerankTaskSettings.defaultsFromMap(null); + var customElandRerankTaskSettings = CustomElandRerankTaskSettings.defaultsFromMap(null); - assertThat(rerankTaskSettings, sameInstance(RerankTaskSettings.DEFAULT_SETTINGS)); + assertThat(customElandRerankTaskSettings, sameInstance(CustomElandRerankTaskSettings.DEFAULT_SETTINGS)); } public void testDefaultsFromMap_MapIsEmpty_ReturnsDefaultSettings() { - var rerankTaskSettings = RerankTaskSettings.defaultsFromMap(new HashMap<>()); + var customElandRerankTaskSettings = CustomElandRerankTaskSettings.defaultsFromMap(new HashMap<>()); - assertThat(rerankTaskSettings, sameInstance(RerankTaskSettings.DEFAULT_SETTINGS)); + assertThat(customElandRerankTaskSettings, sameInstance(CustomElandRerankTaskSettings.DEFAULT_SETTINGS)); } public void testDefaultsFromMap_ExtractedReturnDocumentsNull_SetsReturnDocumentToTrue() { - var rerankTaskSettings = RerankTaskSettings.defaultsFromMap(new HashMap<>()); + var customElandRerankTaskSettings = CustomElandRerankTaskSettings.defaultsFromMap(new HashMap<>()); - assertThat(rerankTaskSettings.returnDocuments(), is(Boolean.TRUE)); + assertThat(customElandRerankTaskSettings.returnDocuments(), is(Boolean.TRUE)); } public void testFromMap_MapIsNull_ReturnsDefaultSettings() { - var rerankTaskSettings = RerankTaskSettings.fromMap(null); + var customElandRerankTaskSettings = CustomElandRerankTaskSettings.fromMap(null); - assertThat(rerankTaskSettings, sameInstance(RerankTaskSettings.DEFAULT_SETTINGS)); + assertThat(customElandRerankTaskSettings, sameInstance(CustomElandRerankTaskSettings.DEFAULT_SETTINGS)); } public void testFromMap_MapIsEmpty_ReturnsDefaultSettings() { - var rerankTaskSettings = RerankTaskSettings.fromMap(new HashMap<>()); + var customElandRerankTaskSettings = CustomElandRerankTaskSettings.fromMap(new HashMap<>()); - assertThat(rerankTaskSettings, sameInstance(RerankTaskSettings.DEFAULT_SETTINGS)); + assertThat(customElandRerankTaskSettings, sameInstance(CustomElandRerankTaskSettings.DEFAULT_SETTINGS)); } public void testToXContent_WritesAllValues() throws IOException { - var serviceSettings = new RerankTaskSettings(Boolean.TRUE); + var serviceSettings = new CustomElandRerankTaskSettings(Boolean.TRUE); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); serviceSettings.toXContent(builder, null); @@ -89,30 +89,30 @@ public void testToXContent_WritesAllValues() throws IOException { } public void testOf_PrefersNonNullRequestTaskSettings() { - var originalSettings = new RerankTaskSettings(Boolean.FALSE); - var requestTaskSettings = new RerankTaskSettings(Boolean.TRUE); + var originalSettings = new CustomElandRerankTaskSettings(Boolean.FALSE); + var requestTaskSettings = new CustomElandRerankTaskSettings(Boolean.TRUE); - var taskSettings = RerankTaskSettings.of(originalSettings, requestTaskSettings); + var taskSettings = CustomElandRerankTaskSettings.of(originalSettings, requestTaskSettings); assertThat(taskSettings, sameInstance(requestTaskSettings)); } - private static RerankTaskSettings createRandom() { - return new RerankTaskSettings(randomOptionalBoolean()); + private static CustomElandRerankTaskSettings createRandom() { + return new CustomElandRerankTaskSettings(randomOptionalBoolean()); } @Override - protected Writeable.Reader instanceReader() { - return RerankTaskSettings::new; + protected Writeable.Reader instanceReader() { + return CustomElandRerankTaskSettings::new; } @Override - protected RerankTaskSettings createTestInstance() { + protected CustomElandRerankTaskSettings createTestInstance() { return createRandom(); } @Override - protected RerankTaskSettings mutateInstance(RerankTaskSettings instance) throws IOException { - return randomValueOtherThan(instance, RerankTaskSettingsTests::createRandom); + protected CustomElandRerankTaskSettings mutateInstance(CustomElandRerankTaskSettings instance) throws IOException { + return randomValueOtherThan(instance, CustomElandRerankTaskSettingsTests::createRandom); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java index 17e6583f11c8f..306509ea60cfc 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java @@ -534,13 +534,16 @@ public void testParseRequestConfig_Rerank() { ) ); var returnDocs = randomBoolean(); - settings.put(ModelConfigurations.TASK_SETTINGS, new HashMap<>(Map.of(RerankTaskSettings.RETURN_DOCUMENTS, returnDocs))); + settings.put( + ModelConfigurations.TASK_SETTINGS, + new HashMap<>(Map.of(CustomElandRerankTaskSettings.RETURN_DOCUMENTS, returnDocs)) + ); ActionListener modelListener = ActionListener.wrap(model -> { assertThat(model, instanceOf(CustomElandRerankModel.class)); - assertThat(model.getTaskSettings(), instanceOf(RerankTaskSettings.class)); + assertThat(model.getTaskSettings(), instanceOf(CustomElandRerankTaskSettings.class)); assertThat(model.getServiceSettings(), instanceOf(CustomElandInternalServiceSettings.class)); - assertEquals(returnDocs, ((RerankTaskSettings) model.getTaskSettings()).returnDocuments()); + assertEquals(returnDocs, ((CustomElandRerankTaskSettings) model.getTaskSettings()).returnDocuments()); }, e -> { fail("Model parsing failed " + e.getMessage()); }); service.parseRequestConfig(randomInferenceEntityId, TaskType.RERANK, settings, modelListener); @@ -580,9 +583,9 @@ public void testParseRequestConfig_Rerank_DefaultTaskSettings() { ActionListener modelListener = ActionListener.wrap(model -> { assertThat(model, instanceOf(CustomElandRerankModel.class)); - assertThat(model.getTaskSettings(), instanceOf(RerankTaskSettings.class)); + assertThat(model.getTaskSettings(), instanceOf(CustomElandRerankTaskSettings.class)); assertThat(model.getServiceSettings(), instanceOf(CustomElandInternalServiceSettings.class)); - assertEquals(Boolean.TRUE, ((RerankTaskSettings) model.getTaskSettings()).returnDocuments()); + assertEquals(Boolean.TRUE, ((CustomElandRerankTaskSettings) model.getTaskSettings()).returnDocuments()); }, e -> { fail("Model parsing failed " + e.getMessage()); }); service.parseRequestConfig(randomInferenceEntityId, TaskType.RERANK, settings, modelListener); @@ -1246,11 +1249,14 @@ public void testParsePersistedConfig_Rerank() { ); settings.put(ElasticsearchInternalServiceSettings.MODEL_ID, "foo"); var returnDocs = randomBoolean(); - settings.put(ModelConfigurations.TASK_SETTINGS, new HashMap<>(Map.of(RerankTaskSettings.RETURN_DOCUMENTS, returnDocs))); + settings.put( + ModelConfigurations.TASK_SETTINGS, + new HashMap<>(Map.of(CustomElandRerankTaskSettings.RETURN_DOCUMENTS, returnDocs)) + ); var model = service.parsePersistedConfig(randomInferenceEntityId, TaskType.RERANK, settings); - assertThat(model.getTaskSettings(), instanceOf(RerankTaskSettings.class)); - assertEquals(returnDocs, ((RerankTaskSettings) model.getTaskSettings()).returnDocuments()); + assertThat(model.getTaskSettings(), instanceOf(CustomElandRerankTaskSettings.class)); + assertEquals(returnDocs, ((CustomElandRerankTaskSettings) model.getTaskSettings()).returnDocuments()); } // without task settings @@ -1273,8 +1279,8 @@ public void testParsePersistedConfig_Rerank() { settings.put(ElasticsearchInternalServiceSettings.MODEL_ID, "foo"); var model = service.parsePersistedConfig(randomInferenceEntityId, TaskType.RERANK, settings); - assertThat(model.getTaskSettings(), instanceOf(RerankTaskSettings.class)); - assertTrue(((RerankTaskSettings) model.getTaskSettings()).returnDocuments()); + assertThat(model.getTaskSettings(), instanceOf(CustomElandRerankTaskSettings.class)); + assertTrue(((CustomElandRerankTaskSettings) model.getTaskSettings()).returnDocuments()); } } @@ -1329,7 +1335,7 @@ private CustomElandModel getCustomElandModel(TaskType taskType) { taskType, ElasticsearchInternalService.NAME, new CustomElandInternalServiceSettings(1, 4, "custom-model", null), - RerankTaskSettings.DEFAULT_SETTINGS + CustomElandRerankTaskSettings.DEFAULT_SETTINGS ); } else if (taskType == TaskType.TEXT_EMBEDDING) { var serviceSettings = new CustomElandInternalTextEmbeddingServiceSettings(1, 4, "custom-model", null); @@ -1522,30 +1528,20 @@ public void testEmbeddingTypeFromTaskTypeAndSettings() { ) ); - var e1 = expectThrows( + var e = expectThrows( ElasticsearchStatusException.class, () -> ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( TaskType.COMPLETION, new ElasticsearchInternalServiceSettings(1, 1, "foo", null) ) ); - assertThat(e1.getMessage(), containsString("Chunking is not supported for task type [completion]")); - - var e2 = expectThrows( - ElasticsearchStatusException.class, - () -> ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( - TaskType.RERANK, - new ElasticsearchInternalServiceSettings(1, 1, "foo", null) - ) - ); - assertThat(e2.getMessage(), containsString("Chunking is not supported for task type [rerank]")); + assertThat(e.getMessage(), containsString("Chunking is not supported for task type [completion]")); } public void testIsDefaultId() { var service = createService(mock(Client.class)); assertTrue(service.isDefaultId(".elser-2-elasticsearch")); assertTrue(service.isDefaultId(".multilingual-e5-small-elasticsearch")); - assertTrue(service.isDefaultId(".rerank-v1-elasticsearch")); assertFalse(service.isDefaultId("foo")); } From ec66857ca13e2f5e7f9088a30aa48ea5ddab17fa Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Mon, 9 Dec 2024 10:03:14 +0000 Subject: [PATCH 116/204] Remove pre-7.2 token serialization support (#118057) --- .../org/elasticsearch/TransportVersions.java | 2 - .../security/SecurityFeatureSetUsage.java | 12 +- .../support/TokensInvalidationResult.java | 6 - .../security/authc/TokenAuthIntegTests.java | 37 ++- .../xpack/security/authc/TokenService.java | 236 +++++------------ .../security/authc/TokenServiceTests.java | 241 +----------------- 6 files changed, 88 insertions(+), 446 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 1a1219825bbbe..40a209c5f0f14 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -54,11 +54,9 @@ static TransportVersion def(int id) { public static final TransportVersion ZERO = def(0); public static final TransportVersion V_7_0_0 = def(7_00_00_99); public static final TransportVersion V_7_0_1 = def(7_00_01_99); - public static final TransportVersion V_7_1_0 = def(7_01_00_99); public static final TransportVersion V_7_2_0 = def(7_02_00_99); public static final TransportVersion V_7_2_1 = def(7_02_01_99); public static final TransportVersion V_7_3_0 = def(7_03_00_99); - public static final TransportVersion V_7_3_2 = def(7_03_02_99); public static final TransportVersion V_7_4_0 = def(7_04_00_99); public static final TransportVersion V_7_5_0 = def(7_05_00_99); public static final TransportVersion V_7_6_0 = def(7_06_00_99); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java index 2793ddea3bd06..33f1a9a469b69 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java @@ -55,10 +55,8 @@ public SecurityFeatureSetUsage(StreamInput in) throws IOException { realmsUsage = in.readGenericMap(); rolesStoreUsage = in.readGenericMap(); sslUsage = in.readGenericMap(); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) { - tokenServiceUsage = in.readGenericMap(); - apiKeyServiceUsage = in.readGenericMap(); - } + tokenServiceUsage = in.readGenericMap(); + apiKeyServiceUsage = in.readGenericMap(); auditUsage = in.readGenericMap(); ipFilterUsage = in.readGenericMap(); anonymousUsage = in.readGenericMap(); @@ -125,10 +123,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeGenericMap(realmsUsage); out.writeGenericMap(rolesStoreUsage); out.writeGenericMap(sslUsage); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) { - out.writeGenericMap(tokenServiceUsage); - out.writeGenericMap(apiKeyServiceUsage); - } + out.writeGenericMap(tokenServiceUsage); + out.writeGenericMap(apiKeyServiceUsage); out.writeGenericMap(auditUsage); out.writeGenericMap(ipFilterUsage); out.writeGenericMap(anonymousUsage); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java index 8fe018a825468..59c16fc8a7a72 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java @@ -59,9 +59,6 @@ public TokensInvalidationResult(StreamInput in) throws IOException { this.invalidatedTokens = in.readStringCollectionAsList(); this.previouslyInvalidatedTokens = in.readStringCollectionAsList(); this.errors = in.readCollectionAsList(StreamInput::readException); - if (in.getTransportVersion().before(TransportVersions.V_7_2_0)) { - in.readVInt(); - } if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_0_0)) { this.restStatus = RestStatus.readFrom(in); } @@ -111,9 +108,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringCollection(invalidatedTokens); out.writeStringCollection(previouslyInvalidatedTokens); out.writeCollection(errors, StreamOutput::writeException); - if (out.getTransportVersion().before(TransportVersions.V_7_2_0)) { - out.writeVInt(5); - } if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_0_0)) { RestStatus.writeTo(out, restStatus); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java index fef1a98ca67e9..b56ea7ae3e456 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java @@ -327,8 +327,8 @@ public void testInvalidateNotValidAccessTokens() throws Exception { ResponseException.class, () -> invalidateAccessToken( tokenService.prependVersionAndEncodeAccessToken( - TransportVersions.V_7_3_2, - tokenService.getRandomTokenBytes(TransportVersions.V_7_3_2, randomBoolean()).v1() + TransportVersions.MINIMUM_COMPATIBLE, + tokenService.getRandomTokenBytes(TransportVersions.MINIMUM_COMPATIBLE, randomBoolean()).v1() ) ) ); @@ -347,7 +347,7 @@ public void testInvalidateNotValidAccessTokens() throws Exception { byte[] longerAccessToken = new byte[randomIntBetween(17, 24)]; random().nextBytes(longerAccessToken); invalidateResponse = invalidateAccessToken( - tokenService.prependVersionAndEncodeAccessToken(TransportVersions.V_7_3_2, longerAccessToken) + tokenService.prependVersionAndEncodeAccessToken(TransportVersions.MINIMUM_COMPATIBLE, longerAccessToken) ); assertThat(invalidateResponse.invalidated(), equalTo(0)); assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0)); @@ -365,7 +365,7 @@ public void testInvalidateNotValidAccessTokens() throws Exception { byte[] shorterAccessToken = new byte[randomIntBetween(12, 15)]; random().nextBytes(shorterAccessToken); invalidateResponse = invalidateAccessToken( - tokenService.prependVersionAndEncodeAccessToken(TransportVersions.V_7_3_2, shorterAccessToken) + tokenService.prependVersionAndEncodeAccessToken(TransportVersions.MINIMUM_COMPATIBLE, shorterAccessToken) ); assertThat(invalidateResponse.invalidated(), equalTo(0)); assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0)); @@ -394,8 +394,8 @@ public void testInvalidateNotValidAccessTokens() throws Exception { invalidateResponse = invalidateAccessToken( tokenService.prependVersionAndEncodeAccessToken( - TransportVersions.V_7_3_2, - tokenService.getRandomTokenBytes(TransportVersions.V_7_3_2, randomBoolean()).v1() + TransportVersions.MINIMUM_COMPATIBLE, + tokenService.getRandomTokenBytes(TransportVersions.MINIMUM_COMPATIBLE, randomBoolean()).v1() ) ); assertThat(invalidateResponse.invalidated(), equalTo(0)); @@ -420,8 +420,8 @@ public void testInvalidateNotValidRefreshTokens() throws Exception { ResponseException.class, () -> invalidateRefreshToken( TokenService.prependVersionAndEncodeRefreshToken( - TransportVersions.V_7_3_2, - tokenService.getRandomTokenBytes(TransportVersions.V_7_3_2, true).v2() + TransportVersions.MINIMUM_COMPATIBLE, + tokenService.getRandomTokenBytes(TransportVersions.MINIMUM_COMPATIBLE, true).v2() ) ) ); @@ -441,7 +441,7 @@ public void testInvalidateNotValidRefreshTokens() throws Exception { byte[] longerRefreshToken = new byte[randomIntBetween(17, 24)]; random().nextBytes(longerRefreshToken); invalidateResponse = invalidateRefreshToken( - TokenService.prependVersionAndEncodeRefreshToken(TransportVersions.V_7_3_2, longerRefreshToken) + TokenService.prependVersionAndEncodeRefreshToken(TransportVersions.MINIMUM_COMPATIBLE, longerRefreshToken) ); assertThat(invalidateResponse.invalidated(), equalTo(0)); assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0)); @@ -459,7 +459,7 @@ public void testInvalidateNotValidRefreshTokens() throws Exception { byte[] shorterRefreshToken = new byte[randomIntBetween(12, 15)]; random().nextBytes(shorterRefreshToken); invalidateResponse = invalidateRefreshToken( - TokenService.prependVersionAndEncodeRefreshToken(TransportVersions.V_7_3_2, shorterRefreshToken) + TokenService.prependVersionAndEncodeRefreshToken(TransportVersions.MINIMUM_COMPATIBLE, shorterRefreshToken) ); assertThat(invalidateResponse.invalidated(), equalTo(0)); assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0)); @@ -488,8 +488,8 @@ public void testInvalidateNotValidRefreshTokens() throws Exception { invalidateResponse = invalidateRefreshToken( TokenService.prependVersionAndEncodeRefreshToken( - TransportVersions.V_7_3_2, - tokenService.getRandomTokenBytes(TransportVersions.V_7_3_2, true).v2() + TransportVersions.MINIMUM_COMPATIBLE, + tokenService.getRandomTokenBytes(TransportVersions.MINIMUM_COMPATIBLE, true).v2() ) ); assertThat(invalidateResponse.invalidated(), equalTo(0)); @@ -758,18 +758,11 @@ public void testAuthenticateWithWrongToken() throws Exception { assertAuthenticateWithToken(response.accessToken(), TEST_USER_NAME); // Now attempt to authenticate with an invalid access token string assertUnauthorizedToken(randomAlphaOfLengthBetween(0, 128)); - // Now attempt to authenticate with an invalid access token with valid structure (pre 7.2) + // Now attempt to authenticate with an invalid access token with valid structure (after 8.0 pre 8.10) assertUnauthorizedToken( tokenService.prependVersionAndEncodeAccessToken( - TransportVersions.V_7_1_0, - tokenService.getRandomTokenBytes(TransportVersions.V_7_1_0, randomBoolean()).v1() - ) - ); - // Now attempt to authenticate with an invalid access token with valid structure (after 7.2 pre 8.10) - assertUnauthorizedToken( - tokenService.prependVersionAndEncodeAccessToken( - TransportVersions.V_7_4_0, - tokenService.getRandomTokenBytes(TransportVersions.V_7_4_0, randomBoolean()).v1() + TransportVersions.V_8_0_0, + tokenService.getRandomTokenBytes(TransportVersions.V_8_0_0, randomBoolean()).v1() ) ); // Now attempt to authenticate with an invalid access token with valid structure (current version) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 4f7ba7808b823..900436a1fd874 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -48,9 +48,7 @@ import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.InputStreamStreamInput; -import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -59,7 +57,6 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Streams; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -93,10 +90,8 @@ import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.IOException; -import java.io.OutputStream; import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -132,7 +127,6 @@ import javax.crypto.Cipher; import javax.crypto.CipherInputStream; -import javax.crypto.CipherOutputStream; import javax.crypto.NoSuchPaddingException; import javax.crypto.SecretKey; import javax.crypto.SecretKeyFactory; @@ -201,14 +195,8 @@ public class TokenService { // UUIDs are 16 bytes encoded base64 without padding, therefore the length is (16 / 3) * 4 + ((16 % 3) * 8 + 5) / 6 chars private static final int TOKEN_LENGTH = 22; private static final String TOKEN_DOC_ID_PREFIX = TOKEN_DOC_TYPE + "_"; - static final int LEGACY_MINIMUM_BYTES = VERSION_BYTES + SALT_BYTES + IV_BYTES + 1; static final int MINIMUM_BYTES = VERSION_BYTES + TOKEN_LENGTH + 1; - static final int LEGACY_MINIMUM_BASE64_BYTES = Double.valueOf(Math.ceil((4 * LEGACY_MINIMUM_BYTES) / 3)).intValue(); public static final int MINIMUM_BASE64_BYTES = Double.valueOf(Math.ceil((4 * MINIMUM_BYTES) / 3)).intValue(); - static final TransportVersion VERSION_HASHED_TOKENS = TransportVersions.V_7_2_0; - static final TransportVersion VERSION_TOKENS_INDEX_INTRODUCED = TransportVersions.V_7_2_0; - static final TransportVersion VERSION_ACCESS_TOKENS_AS_UUIDS = TransportVersions.V_7_2_0; - static final TransportVersion VERSION_MULTIPLE_CONCURRENT_REFRESHES = TransportVersions.V_7_2_0; static final TransportVersion VERSION_CLIENT_AUTH_FOR_REFRESH = TransportVersions.V_8_2_0; static final TransportVersion VERSION_GET_TOKEN_DOC_FOR_REFRESH = TransportVersions.V_8_10_X; @@ -273,8 +261,7 @@ public TokenService( /** * Creates an access token and optionally a refresh token as well, based on the provided authentication and metadata with - * auto-generated values. The created tokens are stored in the security index for versions up to - * {@link #VERSION_TOKENS_INDEX_INTRODUCED} and to a specific security tokens index for later versions. + * auto-generated values. The created tokens are stored a specific security tokens index. */ public void createOAuth2Tokens( Authentication authentication, @@ -291,8 +278,7 @@ public void createOAuth2Tokens( /** * Creates an access token and optionally a refresh token as well from predefined values, based on the provided authentication and - * metadata. The created tokens are stored in the security index for versions up to {@link #VERSION_TOKENS_INDEX_INTRODUCED} and to a - * specific security tokens index for later versions. + * metadata. The created tokens are stored in a specific security tokens index. */ // public for testing public void createOAuth2Tokens( @@ -314,21 +300,15 @@ public void createOAuth2Tokens( * * @param accessTokenBytes The predefined seed value for the access token. This will then be *
    - *
  • Encrypted before stored for versions before {@link #VERSION_TOKENS_INDEX_INTRODUCED}
    • - *
  • Hashed before stored for versions after {@link #VERSION_TOKENS_INDEX_INTRODUCED}
    • - *
  • Stored in the security index for versions up to {@link #VERSION_TOKENS_INDEX_INTRODUCED}
    • - *
  • Stored in a specific security tokens index for versions after - * {@link #VERSION_TOKENS_INDEX_INTRODUCED}
    • + *
  • Hashed before stored
    • + *
  • Stored in a specific security tokens index
    • *
  • Prepended with a version ID and Base64 encoded before returned to the caller of the APIs
    • *
* @param refreshTokenBytes The predefined seed value for the access token. This will then be *
    - *
  • Hashed before stored for versions after {@link #VERSION_TOKENS_INDEX_INTRODUCED}
    • - *
  • Stored in the security index for versions up to {@link #VERSION_TOKENS_INDEX_INTRODUCED}
    • - *
  • Stored in a specific security tokens index for versions after - * {@link #VERSION_TOKENS_INDEX_INTRODUCED}
    • - *
  • Prepended with a version ID and encoded with Base64 before returned to the caller of the APIs - * for versions after {@link #VERSION_TOKENS_INDEX_INTRODUCED}
    • + *
  • Hashed before stored
    • + *
  • Stored in a specific security tokens index
    • + *
  • Prepended with a version ID and Base64 encoded before returned to the caller of the APIs
    • *
* @param tokenVersion The version of the nodes with which these tokens will be compatible. * @param authentication The authentication object representing the user for which the tokens are created @@ -384,7 +364,7 @@ private void createOAuth2Tokens( } else { refreshTokenToStore = refreshTokenToReturn = null; } - } else if (tokenVersion.onOrAfter(VERSION_HASHED_TOKENS)) { + } else { assert accessTokenBytes.length == RAW_TOKEN_BYTES_LENGTH; userTokenId = hashTokenString(Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(accessTokenBytes)); accessTokenToStore = null; @@ -395,18 +375,6 @@ private void createOAuth2Tokens( } else { refreshTokenToStore = refreshTokenToReturn = null; } - } else { - assert accessTokenBytes.length == RAW_TOKEN_BYTES_LENGTH; - userTokenId = Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(accessTokenBytes); - accessTokenToStore = null; - if (refreshTokenBytes != null) { - assert refreshTokenBytes.length == RAW_TOKEN_BYTES_LENGTH; - refreshTokenToStore = refreshTokenToReturn = Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString( - refreshTokenBytes - ); - } else { - refreshTokenToStore = refreshTokenToReturn = null; - } } UserToken userToken = new UserToken(userTokenId, tokenVersion, tokenAuth, getExpirationTime(), metadata); tokenDocument = createTokenDocument(userToken, accessTokenToStore, refreshTokenToStore, originatingClientAuth); @@ -419,23 +387,22 @@ private void createOAuth2Tokens( final RefreshPolicy tokenCreationRefreshPolicy = tokenVersion.onOrAfter(VERSION_GET_TOKEN_DOC_FOR_REFRESH) ? RefreshPolicy.NONE : RefreshPolicy.WAIT_UNTIL; - final SecurityIndexManager tokensIndex = getTokensIndexForVersion(tokenVersion); logger.debug( () -> format( "Using refresh policy [%s] when creating token doc [%s] in the security index [%s]", tokenCreationRefreshPolicy, documentId, - tokensIndex.aliasName() + securityTokensIndex.aliasName() ) ); - final IndexRequest indexTokenRequest = client.prepareIndex(tokensIndex.aliasName()) + final IndexRequest indexTokenRequest = client.prepareIndex(securityTokensIndex.aliasName()) .setId(documentId) .setOpType(OpType.CREATE) .setSource(tokenDocument, XContentType.JSON) .setRefreshPolicy(tokenCreationRefreshPolicy) .request(); - tokensIndex.prepareIndexIfNeededThenExecute( - ex -> listener.onFailure(traceLog("prepare tokens index [" + tokensIndex.aliasName() + "]", documentId, ex)), + securityTokensIndex.prepareIndexIfNeededThenExecute( + ex -> listener.onFailure(traceLog("prepare tokens index [" + securityTokensIndex.aliasName() + "]", documentId, ex)), () -> executeAsyncWithOrigin( client, SECURITY_ORIGIN, @@ -554,17 +521,16 @@ private void getTokenDocById( @Nullable String storedRefreshToken, ActionListener listener ) { - final SecurityIndexManager tokensIndex = getTokensIndexForVersion(tokenVersion); - final SecurityIndexManager frozenTokensIndex = tokensIndex.defensiveCopy(); + final SecurityIndexManager frozenTokensIndex = securityTokensIndex.defensiveCopy(); if (frozenTokensIndex.isAvailable(PRIMARY_SHARDS) == false) { - logger.warn("failed to get access token [{}] because index [{}] is not available", tokenId, tokensIndex.aliasName()); + logger.warn("failed to get access token [{}] because index [{}] is not available", tokenId, securityTokensIndex.aliasName()); listener.onFailure(frozenTokensIndex.getUnavailableReason(PRIMARY_SHARDS)); return; } - final GetRequest getRequest = client.prepareGet(tokensIndex.aliasName(), getTokenDocumentId(tokenId)).request(); + final GetRequest getRequest = client.prepareGet(securityTokensIndex.aliasName(), getTokenDocumentId(tokenId)).request(); final Consumer onFailure = ex -> listener.onFailure(traceLog("get token from id", tokenId, ex)); - tokensIndex.checkIndexVersionThenExecute( - ex -> listener.onFailure(traceLog("prepare tokens index [" + tokensIndex.aliasName() + "]", tokenId, ex)), + securityTokensIndex.checkIndexVersionThenExecute( + ex -> listener.onFailure(traceLog("prepare tokens index [" + securityTokensIndex.aliasName() + "]", tokenId, ex)), () -> executeAsyncWithOrigin( client.threadPool().getThreadContext(), SECURITY_ORIGIN, @@ -610,7 +576,11 @@ private void getTokenDocById( // if the index or the shard is not there / available we assume that // the token is not valid if (isShardNotAvailableException(e)) { - logger.warn("failed to get token doc [{}] because index [{}] is not available", tokenId, tokensIndex.aliasName()); + logger.warn( + "failed to get token doc [{}] because index [{}] is not available", + tokenId, + securityTokensIndex.aliasName() + ); } else { logger.error(() -> "failed to get token doc [" + tokenId + "]", e); } @@ -650,7 +620,7 @@ void decodeToken(String token, boolean validateUserToken, ActionListener VERSION_ACCESS_TOKENS_UUIDS cluster if (in.available() < MINIMUM_BYTES) { logger.debug("invalid token, smaller than [{}] bytes", MINIMUM_BYTES); @@ -660,41 +630,6 @@ void decodeToken(String token, boolean validateUserToken, ActionListener { - if (decodeKey != null) { - try { - final Cipher cipher = getDecryptionCipher(iv, decodeKey, version, decodedSalt); - final String tokenId = decryptTokenId(encryptedTokenId, cipher, version); - getAndValidateUserToken(tokenId, version, null, validateUserToken, listener); - } catch (IOException | GeneralSecurityException e) { - // could happen with a token that is not ours - logger.warn("invalid token", e); - listener.onResponse(null); - } - } else { - // could happen with a token that is not ours - listener.onResponse(null); - } - }, listener::onFailure)); - } else { - logger.debug(() -> format("invalid key %s key: %s", passphraseHash, keyCache.cache.keySet())); - listener.onResponse(null); - } } } catch (Exception e) { // could happen with a token that is not ours @@ -852,11 +787,7 @@ private void indexInvalidation( final Set idsOfOlderTokens = new HashSet<>(); boolean anyOlderTokensBeforeRefreshViaGet = false; for (UserToken userToken : userTokens) { - if (userToken.getTransportVersion().onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { - idsOfRecentTokens.add(userToken.getId()); - } else { - idsOfOlderTokens.add(userToken.getId()); - } + idsOfRecentTokens.add(userToken.getId()); anyOlderTokensBeforeRefreshViaGet |= userToken.getTransportVersion().before(VERSION_GET_TOKEN_DOC_FOR_REFRESH); } final RefreshPolicy tokensInvalidationRefreshPolicy = anyOlderTokensBeforeRefreshViaGet @@ -1124,7 +1055,7 @@ private void findTokenFromRefreshToken(String refreshToken, Iterator ); getTokenDocById(userTokenId, version, null, storedRefreshToken, listener); } - } else if (version.onOrAfter(VERSION_HASHED_TOKENS)) { + } else { final String unencodedRefreshToken = in.readString(); if (unencodedRefreshToken.length() != TOKEN_LENGTH) { logger.debug("Decoded refresh token [{}] with version [{}] is invalid.", unencodedRefreshToken, version); @@ -1133,9 +1064,6 @@ private void findTokenFromRefreshToken(String refreshToken, Iterator final String hashedRefreshToken = hashTokenString(unencodedRefreshToken); findTokenFromRefreshToken(hashedRefreshToken, securityTokensIndex, backoff, listener); } - } else { - logger.debug("Unrecognized refresh token version [{}].", version); - listener.onResponse(null); } } catch (IOException e) { logger.debug(() -> "Could not decode refresh token [" + refreshToken + "].", e); @@ -1250,7 +1178,6 @@ private void innerRefresh( return; } final RefreshTokenStatus refreshTokenStatus = checkRefreshResult.v1(); - final SecurityIndexManager refreshedTokenIndex = getTokensIndexForVersion(refreshTokenStatus.getTransportVersion()); if (refreshTokenStatus.isRefreshed()) { logger.debug( "Token document [{}] was recently refreshed, when a new token document was generated. Reusing that result.", @@ -1258,31 +1185,29 @@ private void innerRefresh( ); final Tuple parsedTokens = parseTokensFromDocument(tokenDoc.sourceAsMap(), null); Authentication authentication = parsedTokens.v1().getAuthentication(); - decryptAndReturnSupersedingTokens(refreshToken, refreshTokenStatus, refreshedTokenIndex, authentication, listener); + decryptAndReturnSupersedingTokens(refreshToken, refreshTokenStatus, securityTokensIndex, authentication, listener); } else { final TransportVersion newTokenVersion = getTokenVersionCompatibility(); final Tuple newTokenBytes = getRandomTokenBytes(newTokenVersion, true); final Map updateMap = new HashMap<>(); updateMap.put("refreshed", true); - if (newTokenVersion.onOrAfter(VERSION_MULTIPLE_CONCURRENT_REFRESHES)) { - updateMap.put("refresh_time", clock.instant().toEpochMilli()); - try { - final byte[] iv = getRandomBytes(IV_BYTES); - final byte[] salt = getRandomBytes(SALT_BYTES); - String encryptedAccessAndRefreshToken = encryptSupersedingTokens( - newTokenBytes.v1(), - newTokenBytes.v2(), - refreshToken, - iv, - salt - ); - updateMap.put("superseding.encrypted_tokens", encryptedAccessAndRefreshToken); - updateMap.put("superseding.encryption_iv", Base64.getEncoder().encodeToString(iv)); - updateMap.put("superseding.encryption_salt", Base64.getEncoder().encodeToString(salt)); - } catch (GeneralSecurityException e) { - logger.warn("could not encrypt access token and refresh token string", e); - onFailure.accept(invalidGrantException("could not refresh the requested token")); - } + updateMap.put("refresh_time", clock.instant().toEpochMilli()); + try { + final byte[] iv = getRandomBytes(IV_BYTES); + final byte[] salt = getRandomBytes(SALT_BYTES); + String encryptedAccessAndRefreshToken = encryptSupersedingTokens( + newTokenBytes.v1(), + newTokenBytes.v2(), + refreshToken, + iv, + salt + ); + updateMap.put("superseding.encrypted_tokens", encryptedAccessAndRefreshToken); + updateMap.put("superseding.encryption_iv", Base64.getEncoder().encodeToString(iv)); + updateMap.put("superseding.encryption_salt", Base64.getEncoder().encodeToString(salt)); + } catch (GeneralSecurityException e) { + logger.warn("could not encrypt access token and refresh token string", e); + onFailure.accept(invalidGrantException("could not refresh the requested token")); } assert tokenDoc.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "expected an assigned sequence number"; assert tokenDoc.primaryTerm() != SequenceNumbers.UNASSIGNED_PRIMARY_TERM : "expected an assigned primary term"; @@ -1293,17 +1218,17 @@ private void innerRefresh( "Using refresh policy [%s] when updating token doc [%s] for refresh in the security index [%s]", tokenRefreshUpdateRefreshPolicy, tokenDoc.id(), - refreshedTokenIndex.aliasName() + securityTokensIndex.aliasName() ) ); - final UpdateRequestBuilder updateRequest = client.prepareUpdate(refreshedTokenIndex.aliasName(), tokenDoc.id()) + final UpdateRequestBuilder updateRequest = client.prepareUpdate(securityTokensIndex.aliasName(), tokenDoc.id()) .setDoc("refresh_token", updateMap) .setFetchSource(logger.isDebugEnabled()) .setRefreshPolicy(tokenRefreshUpdateRefreshPolicy) .setIfSeqNo(tokenDoc.seqNo()) .setIfPrimaryTerm(tokenDoc.primaryTerm()); - refreshedTokenIndex.prepareIndexIfNeededThenExecute( - ex -> listener.onFailure(traceLog("prepare index [" + refreshedTokenIndex.aliasName() + "]", ex)), + securityTokensIndex.prepareIndexIfNeededThenExecute( + ex -> listener.onFailure(traceLog("prepare index [" + securityTokensIndex.aliasName() + "]", ex)), () -> executeAsyncWithOrigin( client.threadPool().getThreadContext(), SECURITY_ORIGIN, @@ -1349,7 +1274,7 @@ private void innerRefresh( if (cause instanceof VersionConflictEngineException) { // The document has been updated by another thread, get it again. logger.debug("version conflict while updating document [{}], attempting to get it again", tokenDoc.id()); - getTokenDocAsync(tokenDoc.id(), refreshedTokenIndex, true, new ActionListener<>() { + getTokenDocAsync(tokenDoc.id(), securityTokensIndex, true, new ActionListener<>() { @Override public void onResponse(GetResponse response) { if (response.isExists()) { @@ -1368,7 +1293,7 @@ public void onFailure(Exception e) { logger.info("could not get token document [{}] for refresh, retrying", tokenDoc.id()); client.threadPool() .schedule( - () -> getTokenDocAsync(tokenDoc.id(), refreshedTokenIndex, true, this), + () -> getTokenDocAsync(tokenDoc.id(), securityTokensIndex, true, this), backoff.next(), client.threadPool().generic() ); @@ -1689,17 +1614,13 @@ private static Optional checkMultipleRefreshes( RefreshTokenStatus refreshTokenStatus ) { if (refreshTokenStatus.isRefreshed()) { - if (refreshTokenStatus.getTransportVersion().onOrAfter(VERSION_MULTIPLE_CONCURRENT_REFRESHES)) { - if (refreshRequested.isAfter(refreshTokenStatus.getRefreshInstant().plus(30L, ChronoUnit.SECONDS))) { - return Optional.of(invalidGrantException("token has already been refreshed more than 30 seconds in the past")); - } - if (refreshRequested.isBefore(refreshTokenStatus.getRefreshInstant().minus(30L, ChronoUnit.SECONDS))) { - return Optional.of( - invalidGrantException("token has been refreshed more than 30 seconds in the future, clock skew too great") - ); - } - } else { - return Optional.of(invalidGrantException("token has already been refreshed")); + if (refreshRequested.isAfter(refreshTokenStatus.getRefreshInstant().plus(30L, ChronoUnit.SECONDS))) { + return Optional.of(invalidGrantException("token has already been refreshed more than 30 seconds in the past")); + } + if (refreshRequested.isBefore(refreshTokenStatus.getRefreshInstant().minus(30L, ChronoUnit.SECONDS))) { + return Optional.of( + invalidGrantException("token has been refreshed more than 30 seconds in the future, clock skew too great") + ); } } return Optional.empty(); @@ -1979,21 +1900,6 @@ private void ensureEnabled() { } } - /** - * In version {@code #VERSION_TOKENS_INDEX_INTRODUCED} security tokens were moved into a separate index, away from the other entities in - * the main security index, due to their ephemeral nature. They moved "seamlessly" - without manual user intervention. In this way, new - * tokens are created in the new index, while the existing ones were left in place - to be accessed from the old index - and due to be - * removed automatically by the {@code ExpiredTokenRemover} periodic job. Therefore, in general, when searching for a token we need to - * consider both the new and the old indices. - */ - private SecurityIndexManager getTokensIndexForVersion(TransportVersion version) { - if (version.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) { - return securityTokensIndex; - } else { - return securityMainIndex; - } - } - public TimeValue getExpirationDelay() { return expirationDelay; } @@ -2022,41 +1928,13 @@ public String prependVersionAndEncodeAccessToken(TransportVersion version, byte[ out.writeByteArray(accessTokenBytes); return Base64.getEncoder().encodeToString(out.bytes().toBytesRef().bytes); } - } else if (version.onOrAfter(VERSION_ACCESS_TOKENS_AS_UUIDS)) { + } else { try (BytesStreamOutput out = new BytesStreamOutput(MINIMUM_BASE64_BYTES)) { out.setTransportVersion(version); TransportVersion.writeVersion(version, out); out.writeString(Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(accessTokenBytes)); return Base64.getEncoder().encodeToString(out.bytes().toBytesRef().bytes); } - } else { - // we know that the minimum length is larger than the default of the ByteArrayOutputStream so set the size to this explicitly - try ( - ByteArrayOutputStream os = new ByteArrayOutputStream(LEGACY_MINIMUM_BASE64_BYTES); - OutputStream base64 = Base64.getEncoder().wrap(os); - StreamOutput out = new OutputStreamStreamOutput(base64) - ) { - out.setTransportVersion(version); - KeyAndCache keyAndCache = keyCache.activeKeyCache; - TransportVersion.writeVersion(version, out); - out.writeByteArray(keyAndCache.getSalt().bytes); - out.writeByteArray(keyAndCache.getKeyHash().bytes); - final byte[] initializationVector = getRandomBytes(IV_BYTES); - out.writeByteArray(initializationVector); - try ( - CipherOutputStream encryptedOutput = new CipherOutputStream( - out, - getEncryptionCipher(initializationVector, keyAndCache, version) - ); - StreamOutput encryptedStreamOutput = new OutputStreamStreamOutput(encryptedOutput) - ) { - encryptedStreamOutput.setTransportVersion(version); - encryptedStreamOutput.writeString(Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(accessTokenBytes)); - // StreamOutput needs to be closed explicitly because it wraps CipherOutputStream - encryptedStreamOutput.close(); - return new String(os.toByteArray(), StandardCharsets.UTF_8); - } - } } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 75c2507a1dc5f..702af75141093 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -126,7 +126,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -148,7 +147,6 @@ public class TokenServiceTests extends ESTestCase { private SecurityIndexManager securityMainIndex; private SecurityIndexManager securityTokensIndex; private ClusterService clusterService; - private DiscoveryNode pre72OldNode; private DiscoveryNode pre8500040OldNode; private Settings tokenServiceEnabledSettings = Settings.builder() .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), true) @@ -228,31 +226,12 @@ public void setupClient() { licenseState = mock(MockLicenseState.class); when(licenseState.isAllowed(Security.TOKEN_SERVICE_FEATURE)).thenReturn(true); - if (randomBoolean()) { - // version 7.2 was an "inflection" point in the Token Service development (access_tokens as UUIDS, multiple concurrent - // refreshes, - // tokens docs on a separate index) - pre72OldNode = addAnother7071DataNode(this.clusterService); - } if (randomBoolean()) { // before refresh tokens used GET, i.e. TokenService#VERSION_GET_TOKEN_DOC_FOR_REFRESH pre8500040OldNode = addAnotherPre8500DataNode(this.clusterService); } } - private static DiscoveryNode addAnother7071DataNode(ClusterService clusterService) { - Version version; - TransportVersion transportVersion; - if (randomBoolean()) { - version = Version.V_7_0_0; - transportVersion = TransportVersions.V_7_0_0; - } else { - version = Version.V_7_1_0; - transportVersion = TransportVersions.V_7_1_0; - } - return addAnotherDataNodeWithVersion(clusterService, version, transportVersion); - } - private static DiscoveryNode addAnotherPre8500DataNode(ClusterService clusterService) { Version version; TransportVersion transportVersion; @@ -301,53 +280,6 @@ public static void shutdownThreadpool() { threadPool = null; } - public void testAttachAndGetToken() throws Exception { - TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC()); - // This test only makes sense in mixed clusters with pre v7.2.0 nodes where the Token Service Key is used (to encrypt tokens) - if (null == pre72OldNode) { - pre72OldNode = addAnother7071DataNode(this.clusterService); - } - Authentication authentication = AuthenticationTestHelper.builder() - .user(new User("joe", "admin")) - .realmRef(new RealmRef("native_realm", "native", "node1")) - .build(false); - PlainActionFuture tokenFuture = new PlainActionFuture<>(); - Tuple newTokenBytes = tokenService.getRandomTokenBytes(randomBoolean()); - tokenService.createOAuth2Tokens( - newTokenBytes.v1(), - newTokenBytes.v2(), - authentication, - authentication, - Collections.emptyMap(), - tokenFuture - ); - final String accessToken = tokenFuture.get().getAccessToken(); - assertNotNull(accessToken); - mockGetTokenFromAccessTokenBytes(tokenService, newTokenBytes.v1(), authentication, false, null); - - ThreadContext requestContext = new ThreadContext(Settings.EMPTY); - requestContext.putHeader("Authorization", randomFrom("Bearer ", "BEARER ", "bearer ") + accessToken); - - try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) { - PlainActionFuture future = new PlainActionFuture<>(); - final SecureString bearerToken = Authenticator.extractBearerTokenFromHeader(requestContext); - tokenService.tryAuthenticateToken(bearerToken, future); - UserToken serialized = future.get(); - assertAuthentication(authentication, serialized.getAuthentication()); - } - - try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) { - // verify a second separate token service with its own salt can also verify - TokenService anotherService = createTokenService(tokenServiceEnabledSettings, systemUTC()); - anotherService.refreshMetadata(tokenService.getTokenMetadata()); - PlainActionFuture future = new PlainActionFuture<>(); - final SecureString bearerToken = Authenticator.extractBearerTokenFromHeader(requestContext); - anotherService.tryAuthenticateToken(bearerToken, future); - UserToken fromOtherService = future.get(); - assertAuthentication(authentication, fromOtherService.getAuthentication()); - } - } - public void testInvalidAuthorizationHeader() throws Exception { TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC()); ThreadContext requestContext = new ThreadContext(Settings.EMPTY); @@ -364,89 +296,6 @@ public void testInvalidAuthorizationHeader() throws Exception { } } - public void testPassphraseWorks() throws Exception { - TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC()); - // This test only makes sense in mixed clusters with pre v7.1.0 nodes where the Key is actually used - if (null == pre72OldNode) { - pre72OldNode = addAnother7071DataNode(this.clusterService); - } - Authentication authentication = AuthenticationTestHelper.builder() - .user(new User("joe", "admin")) - .realmRef(new RealmRef("native_realm", "native", "node1")) - .build(false); - PlainActionFuture tokenFuture = new PlainActionFuture<>(); - Tuple newTokenBytes = tokenService.getRandomTokenBytes(randomBoolean()); - tokenService.createOAuth2Tokens( - newTokenBytes.v1(), - newTokenBytes.v2(), - authentication, - authentication, - Collections.emptyMap(), - tokenFuture - ); - final String accessToken = tokenFuture.get().getAccessToken(); - assertNotNull(accessToken); - mockGetTokenFromAccessTokenBytes(tokenService, newTokenBytes.v1(), authentication, false, null); - - ThreadContext requestContext = new ThreadContext(Settings.EMPTY); - storeTokenHeader(requestContext, accessToken); - - try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) { - PlainActionFuture future = new PlainActionFuture<>(); - final SecureString bearerToken = Authenticator.extractBearerTokenFromHeader(requestContext); - tokenService.tryAuthenticateToken(bearerToken, future); - UserToken serialized = future.get(); - assertAuthentication(authentication, serialized.getAuthentication()); - } - - try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) { - // verify a second separate token service with its own passphrase cannot verify - TokenService anotherService = createTokenService(tokenServiceEnabledSettings, systemUTC()); - PlainActionFuture future = new PlainActionFuture<>(); - final SecureString bearerToken = Authenticator.extractBearerTokenFromHeader(requestContext); - anotherService.tryAuthenticateToken(bearerToken, future); - assertNull(future.get()); - } - } - - public void testGetTokenWhenKeyCacheHasExpired() throws Exception { - TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC()); - // This test only makes sense in mixed clusters with pre v7.1.0 nodes where the Key is actually used - if (null == pre72OldNode) { - pre72OldNode = addAnother7071DataNode(this.clusterService); - } - Authentication authentication = AuthenticationTestHelper.builder() - .user(new User("joe", "admin")) - .realmRef(new RealmRef("native_realm", "native", "node1")) - .build(false); - - PlainActionFuture tokenFuture = new PlainActionFuture<>(); - Tuple newTokenBytes = tokenService.getRandomTokenBytes(randomBoolean()); - tokenService.createOAuth2Tokens( - newTokenBytes.v1(), - newTokenBytes.v2(), - authentication, - authentication, - Collections.emptyMap(), - tokenFuture - ); - String accessToken = tokenFuture.get().getAccessToken(); - assertThat(accessToken, notNullValue()); - - tokenService.clearActiveKeyCache(); - - tokenService.createOAuth2Tokens( - newTokenBytes.v1(), - newTokenBytes.v2(), - authentication, - authentication, - Collections.emptyMap(), - tokenFuture - ); - accessToken = tokenFuture.get().getAccessToken(); - assertThat(accessToken, notNullValue()); - } - public void testAuthnWithInvalidatedToken() throws Exception { when(securityMainIndex.indexExists()).thenReturn(true); TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC()); @@ -820,57 +669,6 @@ public void testMalformedRefreshTokens() throws Exception { } } - public void testNonExistingPre72Token() throws Exception { - TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC()); - // mock another random token so that we don't find a token in TokenService#getUserTokenFromId - Authentication authentication = AuthenticationTestHelper.builder() - .user(new User("joe", "admin")) - .realmRef(new RealmRef("native_realm", "native", "node1")) - .build(false); - mockGetTokenFromAccessTokenBytes(tokenService, tokenService.getRandomTokenBytes(randomBoolean()).v1(), authentication, false, null); - ThreadContext requestContext = new ThreadContext(Settings.EMPTY); - storeTokenHeader( - requestContext, - tokenService.prependVersionAndEncodeAccessToken( - TransportVersions.V_7_1_0, - tokenService.getRandomTokenBytes(TransportVersions.V_7_1_0, randomBoolean()).v1() - ) - ); - - try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) { - PlainActionFuture future = new PlainActionFuture<>(); - final SecureString bearerToken = Authenticator.extractBearerTokenFromHeader(requestContext); - tokenService.tryAuthenticateToken(bearerToken, future); - assertNull(future.get()); - } - } - - public void testNonExistingUUIDToken() throws Exception { - TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC()); - // mock another random token so that we don't find a token in TokenService#getUserTokenFromId - Authentication authentication = AuthenticationTestHelper.builder() - .user(new User("joe", "admin")) - .realmRef(new RealmRef("native_realm", "native", "node1")) - .build(false); - mockGetTokenFromAccessTokenBytes(tokenService, tokenService.getRandomTokenBytes(randomBoolean()).v1(), authentication, false, null); - ThreadContext requestContext = new ThreadContext(Settings.EMPTY); - TransportVersion uuidTokenVersion = randomFrom(TransportVersions.V_7_2_0, TransportVersions.V_7_3_2); - storeTokenHeader( - requestContext, - tokenService.prependVersionAndEncodeAccessToken( - uuidTokenVersion, - tokenService.getRandomTokenBytes(uuidTokenVersion, randomBoolean()).v1() - ) - ); - - try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) { - PlainActionFuture future = new PlainActionFuture<>(); - final SecureString bearerToken = Authenticator.extractBearerTokenFromHeader(requestContext); - tokenService.tryAuthenticateToken(bearerToken, future); - assertNull(future.get()); - } - } - public void testNonExistingLatestTokenVersion() throws Exception { TokenService tokenService = createTokenService(tokenServiceEnabledSettings, systemUTC()); // mock another random token so that we don't find a token in TokenService#getUserTokenFromId @@ -925,18 +723,11 @@ public void testIndexNotAvailable() throws Exception { return Void.TYPE; }).when(client).get(any(GetRequest.class), anyActionListener()); - final SecurityIndexManager tokensIndex; - if (pre72OldNode != null) { - tokensIndex = securityMainIndex; - when(securityTokensIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(false); - when(securityTokensIndex.indexExists()).thenReturn(false); - when(securityTokensIndex.defensiveCopy()).thenReturn(securityTokensIndex); - } else { - tokensIndex = securityTokensIndex; - when(securityMainIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(false); - when(securityMainIndex.indexExists()).thenReturn(false); - when(securityMainIndex.defensiveCopy()).thenReturn(securityMainIndex); - } + final SecurityIndexManager tokensIndex = securityTokensIndex; + when(securityMainIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(false); + when(securityMainIndex.indexExists()).thenReturn(false); + when(securityMainIndex.defensiveCopy()).thenReturn(securityMainIndex); + try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) { PlainActionFuture future = new PlainActionFuture<>(); final SecureString bearerToken3 = Authenticator.extractBearerTokenFromHeader(requestContext); @@ -988,7 +779,6 @@ public void testGetAuthenticationWorksWithExpiredUserToken() throws Exception { } public void testSupersedingTokenEncryption() throws Exception { - assumeTrue("Superseding tokens are only created in post 7.2 clusters", pre72OldNode == null); TokenService tokenService = createTokenService(tokenServiceEnabledSettings, Clock.systemUTC()); Authentication authentication = AuthenticationTests.randomAuthentication(null, null); PlainActionFuture tokenFuture = new PlainActionFuture<>(); @@ -1023,13 +813,11 @@ public void testSupersedingTokenEncryption() throws Exception { authentication, tokenFuture ); - if (version.onOrAfter(TokenService.VERSION_ACCESS_TOKENS_AS_UUIDS)) { - // previous versions serialized the access token encrypted and the cipher text was different each time (due to different IVs) - assertThat( - tokenService.prependVersionAndEncodeAccessToken(version, newTokenBytes.v1()), - equalTo(tokenFuture.get().getAccessToken()) - ); - } + + assertThat( + tokenService.prependVersionAndEncodeAccessToken(version, newTokenBytes.v1()), + equalTo(tokenFuture.get().getAccessToken()) + ); assertThat( TokenService.prependVersionAndEncodeRefreshToken(version, newTokenBytes.v2()), equalTo(tokenFuture.get().getRefreshToken()) @@ -1158,10 +946,8 @@ public static String tokenDocIdFromAccessTokenBytes(byte[] accessTokenBytes, Tra MessageDigest userTokenIdDigest = sha256(); userTokenIdDigest.update(accessTokenBytes, RAW_TOKEN_BYTES_LENGTH, RAW_TOKEN_DOC_ID_BYTES_LENGTH); return Base64.getUrlEncoder().withoutPadding().encodeToString(userTokenIdDigest.digest()); - } else if (tokenVersion.onOrAfter(TokenService.VERSION_ACCESS_TOKENS_AS_UUIDS)) { - return TokenService.hashTokenString(Base64.getUrlEncoder().withoutPadding().encodeToString(accessTokenBytes)); } else { - return Base64.getUrlEncoder().withoutPadding().encodeToString(accessTokenBytes); + return TokenService.hashTokenString(Base64.getUrlEncoder().withoutPadding().encodeToString(accessTokenBytes)); } } @@ -1178,12 +964,9 @@ private void mockTokenForRefreshToken( if (userToken.getTransportVersion().onOrAfter(VERSION_GET_TOKEN_DOC_FOR_REFRESH)) { storedAccessToken = Base64.getUrlEncoder().withoutPadding().encodeToString(sha256().digest(accessTokenBytes)); storedRefreshToken = Base64.getUrlEncoder().withoutPadding().encodeToString(sha256().digest(refreshTokenBytes)); - } else if (userToken.getTransportVersion().onOrAfter(TokenService.VERSION_HASHED_TOKENS)) { - storedAccessToken = null; - storedRefreshToken = TokenService.hashTokenString(Base64.getUrlEncoder().withoutPadding().encodeToString(refreshTokenBytes)); } else { storedAccessToken = null; - storedRefreshToken = Base64.getUrlEncoder().withoutPadding().encodeToString(refreshTokenBytes); + storedRefreshToken = TokenService.hashTokenString(Base64.getUrlEncoder().withoutPadding().encodeToString(refreshTokenBytes)); } final RealmRef realmRef = new RealmRef( refreshTokenStatus == null ? randomAlphaOfLength(6) : refreshTokenStatus.getAssociatedRealm(), From 2e9ff9ae66bff80df6e8a2149fdef818a6efe573 Mon Sep 17 00:00:00 2001 From: Lola Date: Mon, 9 Dec 2024 05:06:46 -0500 Subject: [PATCH 117/204] [Cloud Security]Fix Cloud Security Package indices' deletion step error for ilm policy (#116982) * add ilm deletion step permission for the findings index * add back logs-endpoint index * fix tests for reserved role * fix linting issue --- .../store/KibanaOwnedReservedRoleDescriptors.java | 2 ++ .../authz/store/ReservedRolesStoreTests.java | 14 ++++++-------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index cc589b53eaa1a..5e19b26b8f4de 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -331,6 +331,8 @@ static RoleDescriptor kibanaSystem(String name) { ".logs-endpoint.diagnostic.collection-*", "logs-apm-*", "logs-apm.*-*", + "logs-cloud_security_posture.findings-*", + "logs-cloud_security_posture.vulnerabilities-*", "metrics-apm-*", "metrics-apm.*-*", "traces-apm-*", diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index eeffa1db54856..b69b0ece89960 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -1586,10 +1586,8 @@ public void testKibanaSystemRole() { final IndexAbstraction indexAbstraction = mockIndexAbstraction(cspIndex); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(indexAbstraction), is(false)); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(indexAbstraction), is(false)); - assertThat( - kibanaRole.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(indexAbstraction), - is(false) - ); + // Ensure privileges necessary for ILM policies in Cloud Security Posture Package + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(indexAbstraction), is(true)); assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(indexAbstraction), is(true)); assertThat( kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction), @@ -1613,10 +1611,9 @@ public void testKibanaSystemRole() { final IndexAbstraction indexAbstraction = mockIndexAbstraction(cspIndex); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(indexAbstraction), is(false)); assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(indexAbstraction), is(false)); - assertThat( - kibanaRole.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(indexAbstraction), - is(false) - ); + // Ensure privileges necessary for ILM policies in Cloud Security Posture Package + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(indexAbstraction), is(true)); assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(indexAbstraction), is(true)); assertThat( kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction), @@ -1710,6 +1707,7 @@ public void testKibanaSystemRole() { kibanaRole.indices().allowedIndicesMatcher("indices:monitor/" + randomAlphaOfLengthBetween(3, 8)).test(indexAbstraction), is(true) ); + }); // cloud_defend From 6bb0799893f4ee4e6afa7094346fbc51d5203538 Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:37:41 +0100 Subject: [PATCH 118/204] Updates h7 and h8 formatting (#118132) --- .../connector/docs/connectors-box.asciidoc | 12 +++---- .../connectors-content-extraction.asciidoc | 4 +-- .../docs/connectors-dropbox.asciidoc | 16 +++++----- .../connector/docs/connectors-github.asciidoc | 20 ++++++------ .../connector/docs/connectors-ms-sql.asciidoc | 12 +++---- .../docs/connectors-network-drive.asciidoc | 16 +++++----- .../connector/docs/connectors-notion.asciidoc | 32 +++++++++---------- .../docs/connectors-onedrive.asciidoc | 24 +++++++------- .../docs/connectors-postgresql.asciidoc | 24 +++++++------- .../connector/docs/connectors-s3.asciidoc | 4 +-- .../docs/connectors-salesforce.asciidoc | 12 +++---- .../docs/connectors-servicenow.asciidoc | 12 +++---- .../connectors-sharepoint-online.asciidoc | 16 +++++----- 13 files changed, 102 insertions(+), 102 deletions(-) diff --git a/docs/reference/connector/docs/connectors-box.asciidoc b/docs/reference/connector/docs/connectors-box.asciidoc index 07e4308d67c20..3e95f15d16ccd 100644 --- a/docs/reference/connector/docs/connectors-box.asciidoc +++ b/docs/reference/connector/docs/connectors-box.asciidoc @@ -54,7 +54,7 @@ For additional operations, see <>. ====== Box Free Account [discrete#es-connectors-box-create-oauth-custom-app] -======= Create Box User Authentication (OAuth 2.0) Custom App +*Create Box User Authentication (OAuth 2.0) Custom App* You'll need to create an OAuth app in the Box developer console by following these steps: @@ -64,7 +64,7 @@ You'll need to create an OAuth app in the Box developer console by following the 4. Once the app is created, *Client ID* and *Client secret* values are available in the configuration tab. Keep these handy. [discrete#es-connectors-box-connector-generate-a-refresh-token] -======= Generate a refresh Token +*Generate a refresh Token* To generate a refresh token, follow these steps: @@ -97,7 +97,7 @@ Save the refresh token from the response. You'll need this for the connector con ====== Box Enterprise Account [discrete#es-connectors-box-connector-create-box-server-authentication-client-credentials-grant-custom-app] -======= Create Box Server Authentication (Client Credentials Grant) Custom App +*Create Box Server Authentication (Client Credentials Grant) Custom App* 1. Register a new app in the https://app.box.com/developers/console[Box dev console] with custom App and select Server Authentication (Client Credentials Grant). 2. Check following permissions: @@ -224,7 +224,7 @@ For additional operations, see <>. ====== Box Free Account [discrete#es-connectors-box-client-create-oauth-custom-app] -======= Create Box User Authentication (OAuth 2.0) Custom App +*Create Box User Authentication (OAuth 2.0) Custom App* You'll need to create an OAuth app in the Box developer console by following these steps: @@ -234,7 +234,7 @@ You'll need to create an OAuth app in the Box developer console by following the 4. Once the app is created, *Client ID* and *Client secret* values are available in the configuration tab. Keep these handy. [discrete#es-connectors-box-client-connector-generate-a-refresh-token] -======= Generate a refresh Token +*Generate a refresh Token* To generate a refresh token, follow these steps: @@ -267,7 +267,7 @@ Save the refresh token from the response. You'll need this for the connector con ====== Box Enterprise Account [discrete#es-connectors-box-client-connector-create-box-server-authentication-client-credentials-grant-custom-app] -======= Create Box Server Authentication (Client Credentials Grant) Custom App +*Create Box Server Authentication (Client Credentials Grant) Custom App* 1. Register a new app in the https://app.box.com/developers/console[Box dev console] with custom App and select Server Authentication (Client Credentials Grant). 2. Check following permissions: diff --git a/docs/reference/connector/docs/connectors-content-extraction.asciidoc b/docs/reference/connector/docs/connectors-content-extraction.asciidoc index 5d2a9550a7c3c..a87d38c9bf531 100644 --- a/docs/reference/connector/docs/connectors-content-extraction.asciidoc +++ b/docs/reference/connector/docs/connectors-content-extraction.asciidoc @@ -183,7 +183,7 @@ Be aware that the self-managed connector will download files with randomized fil For that reason, we recommend using a dedicated directory for self-hosted extraction. [discrete#es-connectors-content-extraction-data-extraction-service-file-pointers-configuration-example] -======= Example +*Example* 1. For this example, we will be using `/app/files` as both our local directory and our container directory. When you run the extraction service docker container, you can mount the directory as a volume using the command-line option `-v /app/files:/app/files`. @@ -228,7 +228,7 @@ When using self-hosted extraction from a dockerized self-managed connector, ther * The self-managed connector and the extraction service will also need to share a volume. You can decide what directory inside these docker containers the volume will be mounted onto, but the directory must be the same for both docker containers. [discrete#es-connectors-content-extraction-data-extraction-service-file-pointers-configuration-dockerized-example] -======= Example +*Example* 1. First, set up a volume for the two docker containers to share. This will be where files are downloaded into and then extracted from. diff --git a/docs/reference/connector/docs/connectors-dropbox.asciidoc b/docs/reference/connector/docs/connectors-dropbox.asciidoc index 1f80a0ab4e952..295b7e2936625 100644 --- a/docs/reference/connector/docs/connectors-dropbox.asciidoc +++ b/docs/reference/connector/docs/connectors-dropbox.asciidoc @@ -190,7 +190,7 @@ When both are provided, priority is given to `file_categories`. We have some examples below for illustration. [discrete#es-connectors-dropbox-sync-rules-advanced-example-1] -======= Example: Query only +*Example: Query only* [source,js] ---- @@ -206,7 +206,7 @@ We have some examples below for illustration. // NOTCONSOLE [discrete#es-connectors-dropbox-sync-rules-advanced-example-2] -======= Example: Query with file extension filter +*Example: Query with file extension filter* [source,js] ---- @@ -225,7 +225,7 @@ We have some examples below for illustration. // NOTCONSOLE [discrete#es-connectors-dropbox-sync-rules-advanced-example-3] -======= Example: Query with file category filter +*Example: Query with file category filter* [source,js] ---- @@ -248,7 +248,7 @@ We have some examples below for illustration. // NOTCONSOLE [discrete#es-connectors-dropbox-sync-rules-advanced-limitations] -======= Limitations +*Limitations* * Content extraction is not supported for Dropbox *Paper* files when advanced sync rules are enabled. @@ -474,7 +474,7 @@ When both are provided, priority is given to `file_categories`. We have some examples below for illustration. [discrete#es-connectors-dropbox-client-sync-rules-advanced-example-1] -======= Example: Query only +*Example: Query only* [source,js] ---- @@ -490,7 +490,7 @@ We have some examples below for illustration. // NOTCONSOLE [discrete#es-connectors-dropbox-client-sync-rules-advanced-example-2] -======= Example: Query with file extension filter +*Example: Query with file extension filter* [source,js] ---- @@ -509,7 +509,7 @@ We have some examples below for illustration. // NOTCONSOLE [discrete#es-connectors-dropbox-client-sync-rules-advanced-example-3] -======= Example: Query with file category filter +*Example: Query with file category filter* [source,js] ---- @@ -532,7 +532,7 @@ We have some examples below for illustration. // NOTCONSOLE [discrete#es-connectors-dropbox-client-sync-rules-advanced-limitations] -======= Limitations +*Limitations* * Content extraction is not supported for Dropbox *Paper* files when advanced sync rules are enabled. diff --git a/docs/reference/connector/docs/connectors-github.asciidoc b/docs/reference/connector/docs/connectors-github.asciidoc index aa683e4bb0829..df577d83e8121 100644 --- a/docs/reference/connector/docs/connectors-github.asciidoc +++ b/docs/reference/connector/docs/connectors-github.asciidoc @@ -210,7 +210,7 @@ Advanced sync rules are defined through a source-specific DSL JSON snippet. The following sections provide examples of advanced sync rules for this connector. [discrete#es-connectors-github-sync-rules-advanced-branch] -======= Indexing document and files based on branch name configured via branch key +*Indexing document and files based on branch name configured via branch key* [source,js] ---- @@ -226,7 +226,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-github-sync-rules-advanced-issue-key] -======= Indexing document based on issue query related to bugs via issue key +*Indexing document based on issue query related to bugs via issue key* [source,js] ---- @@ -242,7 +242,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-github-sync-rules-advanced-pr-key] -======= Indexing document based on PR query related to open PR's via PR key +*Indexing document based on PR query related to open PR's via PR key* [source,js] ---- @@ -258,7 +258,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-github-sync-rules-advanced-issue-query-branch-name] -======= Indexing document and files based on queries and branch name +*Indexing document and files based on queries and branch name* [source,js] ---- @@ -283,7 +283,7 @@ Check the Elasticsearch index for the actual document count. ==== [discrete#es-connectors-github-sync-rules-advanced-overlapping] -======= Advanced rules for overlapping +*Advanced rules for overlapping* [source,js] ---- @@ -550,7 +550,7 @@ Advanced sync rules are defined through a source-specific DSL JSON snippet. The following sections provide examples of advanced sync rules for this connector. [discrete#es-connectors-github-client-sync-rules-advanced-branch] -======= Indexing document and files based on branch name configured via branch key +*Indexing document and files based on branch name configured via branch key* [source,js] ---- @@ -566,7 +566,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-github-client-sync-rules-advanced-issue-key] -======= Indexing document based on issue query related to bugs via issue key +*Indexing document based on issue query related to bugs via issue key* [source,js] ---- @@ -582,7 +582,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-github-client-sync-rules-advanced-pr-key] -======= Indexing document based on PR query related to open PR's via PR key +*Indexing document based on PR query related to open PR's via PR key* [source,js] ---- @@ -598,7 +598,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-github-client-sync-rules-advanced-issue-query-branch-name] -======= Indexing document and files based on queries and branch name +*Indexing document and files based on queries and branch name* [source,js] ---- @@ -623,7 +623,7 @@ Check the Elasticsearch index for the actual document count. ==== [discrete#es-connectors-github-client-sync-rules-advanced-overlapping] -======= Advanced rules for overlapping +*Advanced rules for overlapping* [source,js] ---- diff --git a/docs/reference/connector/docs/connectors-ms-sql.asciidoc b/docs/reference/connector/docs/connectors-ms-sql.asciidoc index 47fb282b16877..d706af8ca8043 100644 --- a/docs/reference/connector/docs/connectors-ms-sql.asciidoc +++ b/docs/reference/connector/docs/connectors-ms-sql.asciidoc @@ -196,7 +196,7 @@ Here are a few examples of advanced sync rules for this connector. ==== [discrete#es-connectors-ms-sql-sync-rules-advanced-queries] -======= Example: Two queries +*Example: Two queries* These rules fetch all records from both the `employee` and `customer` tables. The data from these tables will be synced separately to Elasticsearch. @@ -220,7 +220,7 @@ These rules fetch all records from both the `employee` and `customer` tables. Th // NOTCONSOLE [discrete#es-connectors-ms-sql-sync-rules-example-one-where] -======= Example: One WHERE query +*Example: One WHERE query* This rule fetches only the records from the `employee` table where the `emp_id` is greater than 5. Only these filtered records will be synced to Elasticsearch. @@ -236,7 +236,7 @@ This rule fetches only the records from the `employee` table where the `emp_id` // NOTCONSOLE [discrete#es-connectors-ms-sql-sync-rules-example-one-join] -======= Example: One JOIN query +*Example: One JOIN query* This rule fetches records by performing an INNER JOIN between the `employee` and `customer` tables on the condition that the `emp_id` in `employee` matches the `c_id` in `customer`. The result of this combined data will be synced to Elasticsearch. @@ -484,7 +484,7 @@ Here are a few examples of advanced sync rules for this connector. ==== [discrete#es-connectors-ms-sql-client-sync-rules-advanced-queries] -======= Example: Two queries +*Example: Two queries* These rules fetch all records from both the `employee` and `customer` tables. The data from these tables will be synced separately to Elasticsearch. @@ -508,7 +508,7 @@ These rules fetch all records from both the `employee` and `customer` tables. Th // NOTCONSOLE [discrete#es-connectors-ms-sql-client-sync-rules-example-one-where] -======= Example: One WHERE query +*Example: One WHERE query* This rule fetches only the records from the `employee` table where the `emp_id` is greater than 5. Only these filtered records will be synced to Elasticsearch. @@ -524,7 +524,7 @@ This rule fetches only the records from the `employee` table where the `emp_id` // NOTCONSOLE [discrete#es-connectors-ms-sql-client-sync-rules-example-one-join] -======= Example: One JOIN query +*Example: One JOIN query* This rule fetches records by performing an INNER JOIN between the `employee` and `customer` tables on the condition that the `emp_id` in `employee` matches the `c_id` in `customer`. The result of this combined data will be synced to Elasticsearch. diff --git a/docs/reference/connector/docs/connectors-network-drive.asciidoc b/docs/reference/connector/docs/connectors-network-drive.asciidoc index 91c9d3b28c385..909e3440c9f02 100644 --- a/docs/reference/connector/docs/connectors-network-drive.asciidoc +++ b/docs/reference/connector/docs/connectors-network-drive.asciidoc @@ -174,7 +174,7 @@ Advanced sync rules for this connector use *glob patterns*. The following sections provide examples of advanced sync rules for this connector. [discrete#es-connectors-network-drive-indexing-files-and-folders-recursively-within-folders] -======= Indexing files and folders recursively within folders +*Indexing files and folders recursively within folders* [source,js] ---- @@ -190,7 +190,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-network-drive-indexing-files-and-folders-directly-inside-folder] -======= Indexing files and folders directly inside folder +*Indexing files and folders directly inside folder* [source,js] ---- @@ -203,7 +203,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-network-drive-indexing-files-and-folders-directly-inside-a-set-of-folders] -======= Indexing files and folders directly inside a set of folders +*Indexing files and folders directly inside a set of folders* [source,js] ---- @@ -216,7 +216,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-network-drive-excluding-files-and-folders-that-match-a-pattern] -======= Excluding files and folders that match a pattern +*Excluding files and folders that match a pattern* [source,js] ---- @@ -432,7 +432,7 @@ Advanced sync rules for this connector use *glob patterns*. The following sections provide examples of advanced sync rules for this connector. [discrete#es-connectors-network-drive-client-indexing-files-and-folders-recursively-within-folders] -======= Indexing files and folders recursively within folders +*Indexing files and folders recursively within folders* [source,js] ---- @@ -448,7 +448,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-network-drive-client-indexing-files-and-folders-directly-inside-folder] -======= Indexing files and folders directly inside folder +*Indexing files and folders directly inside folder* [source,js] ---- @@ -461,7 +461,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-network-drive-client-indexing-files-and-folders-directly-inside-a-set-of-folders] -======= Indexing files and folders directly inside a set of folders +*Indexing files and folders directly inside a set of folders* [source,js] ---- @@ -474,7 +474,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-network-drive-client-excluding-files-and-folders-that-match-a-pattern] -======= Excluding files and folders that match a pattern +*Excluding files and folders that match a pattern* [source,js] ---- diff --git a/docs/reference/connector/docs/connectors-notion.asciidoc b/docs/reference/connector/docs/connectors-notion.asciidoc index 2d7a71bff20de..7c08c5d81e032 100644 --- a/docs/reference/connector/docs/connectors-notion.asciidoc +++ b/docs/reference/connector/docs/connectors-notion.asciidoc @@ -140,7 +140,7 @@ Advanced sync rules for Notion take the following parameters: ====== Examples [discrete] -======= Example 1 +*Example 1* Indexing every page where the title contains `Demo Page`: @@ -160,7 +160,7 @@ Indexing every page where the title contains `Demo Page`: // NOTCONSOLE [discrete] -======= Example 2 +*Example 2* Indexing every database where the title contains `Demo Database`: @@ -180,7 +180,7 @@ Indexing every database where the title contains `Demo Database`: // NOTCONSOLE [discrete] -======= Example 3 +*Example 3* Indexing every database where the title contains `Demo Database` and every page where the title contains `Demo Page`: @@ -206,7 +206,7 @@ Indexing every database where the title contains `Demo Database` and every page // NOTCONSOLE [discrete] -======= Example 4 +*Example 4* Indexing all pages in the workspace: @@ -226,7 +226,7 @@ Indexing all pages in the workspace: // NOTCONSOLE [discrete] -======= Example 5 +*Example 5* Indexing all the pages and databases connected to the workspace: @@ -243,7 +243,7 @@ Indexing all the pages and databases connected to the workspace: // NOTCONSOLE [discrete] -======= Example 6 +*Example 6* Indexing all the rows of a database where the record is `true` for the column `Task completed` and its property(datatype) is a checkbox: @@ -266,7 +266,7 @@ Indexing all the rows of a database where the record is `true` for the column `T // NOTCONSOLE [discrete] -======= Example 7 +*Example 7* Indexing all rows of a specific database: @@ -283,7 +283,7 @@ Indexing all rows of a specific database: // NOTCONSOLE [discrete] -======= Example 8 +*Example 8* Indexing all blocks defined in `searches` and `database_query_filters`: @@ -498,7 +498,7 @@ Advanced sync rules for Notion take the following parameters: ====== Examples [discrete] -======= Example 1 +*Example 1* Indexing every page where the title contains `Demo Page`: @@ -518,7 +518,7 @@ Indexing every page where the title contains `Demo Page`: // NOTCONSOLE [discrete] -======= Example 2 +*Example 2* Indexing every database where the title contains `Demo Database`: @@ -538,7 +538,7 @@ Indexing every database where the title contains `Demo Database`: // NOTCONSOLE [discrete] -======= Example 3 +*Example 3* Indexing every database where the title contains `Demo Database` and every page where the title contains `Demo Page`: @@ -564,7 +564,7 @@ Indexing every database where the title contains `Demo Database` and every page // NOTCONSOLE [discrete] -======= Example 4 +*Example 4* Indexing all pages in the workspace: @@ -584,7 +584,7 @@ Indexing all pages in the workspace: // NOTCONSOLE [discrete] -======= Example 5 +*Example 5* Indexing all the pages and databases connected to the workspace: @@ -601,7 +601,7 @@ Indexing all the pages and databases connected to the workspace: // NOTCONSOLE [discrete] -======= Example 6 +*Example 6* Indexing all the rows of a database where the record is `true` for the column `Task completed` and its property(datatype) is a checkbox: @@ -624,7 +624,7 @@ Indexing all the rows of a database where the record is `true` for the column `T // NOTCONSOLE [discrete] -======= Example 7 +*Example 7* Indexing all rows of a specific database: @@ -641,7 +641,7 @@ Indexing all rows of a specific database: // NOTCONSOLE [discrete] -======= Example 8 +*Example 8* Indexing all blocks defined in `searches` and `database_query_filters`: diff --git a/docs/reference/connector/docs/connectors-onedrive.asciidoc b/docs/reference/connector/docs/connectors-onedrive.asciidoc index 7d1a21aeb78db..44ac96e2ad99d 100644 --- a/docs/reference/connector/docs/connectors-onedrive.asciidoc +++ b/docs/reference/connector/docs/connectors-onedrive.asciidoc @@ -160,7 +160,7 @@ A <> is required for advanced sync rul Here are a few examples of advanced sync rules for this connector. [discrete#es-connectors-onedrive-sync-rules-advanced-examples-1] -======= Example 1 +*Example 1* This rule skips indexing for files with `.xlsx` and `.docx` extensions. All other files and folders will be indexed. @@ -176,7 +176,7 @@ All other files and folders will be indexed. // NOTCONSOLE [discrete#es-connectors-onedrive-sync-rules-advanced-examples-2] -======= Example 2 +*Example 2* This rule focuses on indexing files and folders owned by `user1-domain@onmicrosoft.com` and `user2-domain@onmicrosoft.com` but excludes files with `.py` extension. @@ -192,7 +192,7 @@ This rule focuses on indexing files and folders owned by `user1-domain@onmicroso // NOTCONSOLE [discrete#es-connectors-onedrive-sync-rules-advanced-examples-3] -======= Example 3 +*Example 3* This rule indexes only the files and folders directly inside the root folder, excluding any `.md` files. @@ -208,7 +208,7 @@ This rule indexes only the files and folders directly inside the root folder, ex // NOTCONSOLE [discrete#es-connectors-onedrive-sync-rules-advanced-examples-4] -======= Example 4 +*Example 4* This rule indexes files and folders owned by `user1-domain@onmicrosoft.com` and `user3-domain@onmicrosoft.com` that are directly inside the `abc` folder, which is a subfolder of any folder under the `hello` directory in the root. Files with extensions `.pdf` and `.py` are excluded. @@ -225,7 +225,7 @@ This rule indexes files and folders owned by `user1-domain@onmicrosoft.com` and // NOTCONSOLE [discrete#es-connectors-onedrive-sync-rules-advanced-examples-5] -======= Example 5 +*Example 5* This example contains two rules. The first rule indexes all files and folders owned by `user1-domain@onmicrosoft.com` and `user2-domain@onmicrosoft.com`. @@ -245,7 +245,7 @@ The second rule indexes files for all other users, but skips files with a `.py` // NOTCONSOLE [discrete#es-connectors-onedrive-sync-rules-advanced-examples-6] -======= Example 6 +*Example 6* This example contains two rules. The first rule indexes all files owned by `user1-domain@onmicrosoft.com` and `user2-domain@onmicrosoft.com`, excluding `.md` files. @@ -449,7 +449,7 @@ A <> is required for advanced sync rul Here are a few examples of advanced sync rules for this connector. [discrete#es-connectors-onedrive-client-sync-rules-advanced-examples-1] -======= Example 1 +*Example 1* This rule skips indexing for files with `.xlsx` and `.docx` extensions. All other files and folders will be indexed. @@ -465,7 +465,7 @@ All other files and folders will be indexed. // NOTCONSOLE [discrete#es-connectors-onedrive-client-sync-rules-advanced-examples-2] -======= Example 2 +*Example 2* This rule focuses on indexing files and folders owned by `user1-domain@onmicrosoft.com` and `user2-domain@onmicrosoft.com` but excludes files with `.py` extension. @@ -481,7 +481,7 @@ This rule focuses on indexing files and folders owned by `user1-domain@onmicroso // NOTCONSOLE [discrete#es-connectors-onedrive-client-sync-rules-advanced-examples-3] -======= Example 3 +*Example 3* This rule indexes only the files and folders directly inside the root folder, excluding any `.md` files. @@ -497,7 +497,7 @@ This rule indexes only the files and folders directly inside the root folder, ex // NOTCONSOLE [discrete#es-connectors-onedrive-client-sync-rules-advanced-examples-4] -======= Example 4 +*Example 4* This rule indexes files and folders owned by `user1-domain@onmicrosoft.com` and `user3-domain@onmicrosoft.com` that are directly inside the `abc` folder, which is a subfolder of any folder under the `hello` directory in the root. Files with extensions `.pdf` and `.py` are excluded. @@ -514,7 +514,7 @@ This rule indexes files and folders owned by `user1-domain@onmicrosoft.com` and // NOTCONSOLE [discrete#es-connectors-onedrive-client-sync-rules-advanced-examples-5] -======= Example 5 +*Example 5* This example contains two rules. The first rule indexes all files and folders owned by `user1-domain@onmicrosoft.com` and `user2-domain@onmicrosoft.com`. @@ -534,7 +534,7 @@ The second rule indexes files for all other users, but skips files with a `.py` // NOTCONSOLE [discrete#es-connectors-onedrive-client-sync-rules-advanced-examples-6] -======= Example 6 +*Example 6* This example contains two rules. The first rule indexes all files owned by `user1-domain@onmicrosoft.com` and `user2-domain@onmicrosoft.com`, excluding `.md` files. diff --git a/docs/reference/connector/docs/connectors-postgresql.asciidoc b/docs/reference/connector/docs/connectors-postgresql.asciidoc index 1fe28f867337c..aa6cb7f29e633 100644 --- a/docs/reference/connector/docs/connectors-postgresql.asciidoc +++ b/docs/reference/connector/docs/connectors-postgresql.asciidoc @@ -188,7 +188,7 @@ Advanced sync rules are defined through a source-specific DSL JSON snippet. Here is some example data that will be used in the following examples. [discrete#connectors-postgresql-sync-rules-advanced-example-data-1] -======= `employee` table +*`employee` table* [cols="3*", options="header"] |=== @@ -199,7 +199,7 @@ Here is some example data that will be used in the following examples. |=== [discrete#connectors-postgresql-sync-rules-advanced-example-2] -======= `customer` table +*`customer` table* [cols="3*", options="header"] |=== @@ -213,7 +213,7 @@ Here is some example data that will be used in the following examples. ====== Advanced sync rules examples [discrete#connectors-postgresql-sync-rules-advanced-examples-1] -======= Multiple table queries +*Multiple table queries* [source,js] ---- @@ -235,7 +235,7 @@ Here is some example data that will be used in the following examples. // NOTCONSOLE [discrete#connectors-postgresql-sync-rules-advanced-examples-1-id-columns] -======= Multiple table queries with `id_columns` +*Multiple table queries with `id_columns`* In 8.15.0, we added a new optional `id_columns` field in our advanced sync rules for the PostgreSQL connector. Use the `id_columns` field to ingest tables which do not have a primary key. Include the names of unique fields so that the connector can use them to generate unique IDs for documents. @@ -264,7 +264,7 @@ Use the `id_columns` field to ingest tables which do not have a primary key. Inc This example uses the `id_columns` field to specify the unique fields `emp_id` and `c_id` for the `employee` and `customer` tables, respectively. [discrete#connectors-postgresql-sync-rules-advanced-examples-2] -======= Filtering data with `WHERE` clause +*Filtering data with `WHERE` clause* [source,js] ---- @@ -278,7 +278,7 @@ This example uses the `id_columns` field to specify the unique fields `emp_id` a // NOTCONSOLE [discrete#connectors-postgresql-sync-rules-advanced-examples-3] -======= `JOIN` operations +*`JOIN` operations* [source,js] ---- @@ -494,7 +494,7 @@ Advanced sync rules are defined through a source-specific DSL JSON snippet. Here is some example data that will be used in the following examples. [discrete#es-connectors-postgresql-client-sync-rules-advanced-example-data-1] -======= `employee` table +*`employee` table* [cols="3*", options="header"] |=== @@ -505,7 +505,7 @@ Here is some example data that will be used in the following examples. |=== [discrete#es-connectors-postgresql-client-sync-rules-advanced-example-2] -======= `customer` table +*`customer` table* [cols="3*", options="header"] |=== @@ -519,7 +519,7 @@ Here is some example data that will be used in the following examples. ====== Advanced sync rules examples [discrete#es-connectors-postgresql-client-sync-rules-advanced-examples-1] -======== Multiple table queries +*Multiple table queries* [source,js] ---- @@ -541,7 +541,7 @@ Here is some example data that will be used in the following examples. // NOTCONSOLE [discrete#es-connectors-postgresql-client-sync-rules-advanced-examples-1-id-columns] -======== Multiple table queries with `id_columns` +*Multiple table queries with `id_columns`* In 8.15.0, we added a new optional `id_columns` field in our advanced sync rules for the PostgreSQL connector. Use the `id_columns` field to ingest tables which do not have a primary key. Include the names of unique fields so that the connector can use them to generate unique IDs for documents. @@ -570,7 +570,7 @@ Use the `id_columns` field to ingest tables which do not have a primary key. Inc This example uses the `id_columns` field to specify the unique fields `emp_id` and `c_id` for the `employee` and `customer` tables, respectively. [discrete#es-connectors-postgresql-client-sync-rules-advanced-examples-2] -======== Filtering data with `WHERE` clause +*Filtering data with `WHERE` clause* [source,js] ---- @@ -584,7 +584,7 @@ This example uses the `id_columns` field to specify the unique fields `emp_id` a // NOTCONSOLE [discrete#es-connectors-postgresql-client-sync-rules-advanced-examples-3] -======== `JOIN` operations +*`JOIN` operations* [source,js] ---- diff --git a/docs/reference/connector/docs/connectors-s3.asciidoc b/docs/reference/connector/docs/connectors-s3.asciidoc index b4d08d3884631..90c070f7b8044 100644 --- a/docs/reference/connector/docs/connectors-s3.asciidoc +++ b/docs/reference/connector/docs/connectors-s3.asciidoc @@ -118,7 +118,7 @@ The connector will fetch file and folder data that matches the string. Defaults to `""` (syncs all bucket objects). [discrete#es-connectors-s3-sync-rules-advanced-examples] -======= Advanced sync rules examples +*Advanced sync rules examples* *Fetching files and folders recursively by prefix* @@ -336,7 +336,7 @@ The connector will fetch file and folder data that matches the string. Defaults to `""` (syncs all bucket objects). [discrete#es-connectors-s3-client-sync-rules-advanced-examples] -======= Advanced sync rules examples +*Advanced sync rules examples* *Fetching files and folders recursively by prefix* diff --git a/docs/reference/connector/docs/connectors-salesforce.asciidoc b/docs/reference/connector/docs/connectors-salesforce.asciidoc index 3676f7663089c..c640751de92c0 100644 --- a/docs/reference/connector/docs/connectors-salesforce.asciidoc +++ b/docs/reference/connector/docs/connectors-salesforce.asciidoc @@ -227,7 +227,7 @@ They take the following parameters: Allowed values are *SOQL* and *SOSL*. [discrete#es-connectors-salesforce-sync-rules-advanced-fetch-query-language] -======= Fetch documents based on the query and language specified +*Fetch documents based on the query and language specified* **Example**: Fetch documents using SOQL query @@ -256,7 +256,7 @@ Allowed values are *SOQL* and *SOSL*. // NOTCONSOLE [discrete#es-connectors-salesforce-sync-rules-advanced-fetch-objects] -======= Fetch standard and custom objects using SOQL and SOSL queries +*Fetch standard and custom objects using SOQL and SOSL queries* **Example**: Fetch documents for standard objects via SOQL and SOSL query. @@ -293,7 +293,7 @@ Allowed values are *SOQL* and *SOSL*. // NOTCONSOLE [discrete#es-connectors-salesforce-sync-rules-advanced-fetch-standard-custom-fields] -======= Fetch documents with standard and custom fields +*Fetch documents with standard and custom fields* **Example**: Fetch documents with all standard and custom fields for Account object. @@ -626,7 +626,7 @@ They take the following parameters: Allowed values are *SOQL* and *SOSL*. [discrete#es-connectors-salesforce-client-sync-rules-advanced-fetch-query-language] -======= Fetch documents based on the query and language specified +*Fetch documents based on the query and language specified* **Example**: Fetch documents using SOQL query @@ -655,7 +655,7 @@ Allowed values are *SOQL* and *SOSL*. // NOTCONSOLE [discrete#es-connectors-salesforce-client-sync-rules-advanced-fetch-objects] -======= Fetch standard and custom objects using SOQL and SOSL queries +*Fetch standard and custom objects using SOQL and SOSL queries* **Example**: Fetch documents for standard objects via SOQL and SOSL query. @@ -692,7 +692,7 @@ Allowed values are *SOQL* and *SOSL*. // NOTCONSOLE [discrete#es-connectors-salesforce-client-sync-rules-advanced-fetch-standard-custom-fields] -======= Fetch documents with standard and custom fields +*Fetch documents with standard and custom fields* **Example**: Fetch documents with all standard and custom fields for Account object. diff --git a/docs/reference/connector/docs/connectors-servicenow.asciidoc b/docs/reference/connector/docs/connectors-servicenow.asciidoc index a02c418f11d74..3dc98ed9a44c9 100644 --- a/docs/reference/connector/docs/connectors-servicenow.asciidoc +++ b/docs/reference/connector/docs/connectors-servicenow.asciidoc @@ -167,7 +167,7 @@ Advanced sync rules are defined through a source-specific DSL JSON snippet. The following sections provide examples of advanced sync rules for this connector. [discrete#es-connectors-servicenow-sync-rules-number-incident-service] -======= Indexing document based on incident number for Incident service +*Indexing document based on incident number for Incident service* [source,js] ---- @@ -181,7 +181,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-servicenow-sync-rules-active-false-user-service] -======= Indexing document based on user activity state for User service +*Indexing document based on user activity state for User service* [source,js] ---- @@ -195,7 +195,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-servicenow-sync-rules-author-administrator-knowledge-service] -======= Indexing document based on author name for Knowledge service +*Indexing document based on author name for Knowledge service* [source,js] ---- @@ -407,7 +407,7 @@ Advanced sync rules are defined through a source-specific DSL JSON snippet. The following sections provide examples of advanced sync rules for this connector. [discrete#es-connectors-servicenow-client-sync-rules-number-incident-service] -======= Indexing document based on incident number for Incident service +*Indexing document based on incident number for Incident service* [source,js] ---- @@ -421,7 +421,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-servicenow-client-sync-rules-active-false-user-service] -======= Indexing document based on user activity state for User service +*Indexing document based on user activity state for User service* [source,js] ---- @@ -435,7 +435,7 @@ The following sections provide examples of advanced sync rules for this connecto // NOTCONSOLE [discrete#es-connectors-servicenow-client-sync-rules-author-administrator-knowledge-service] -======= Indexing document based on author name for Knowledge service +*Indexing document based on author name for Knowledge service* [source,js] ---- diff --git a/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc b/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc index 21d0890e436c5..02f598c16f63c 100644 --- a/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc +++ b/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc @@ -277,7 +277,7 @@ Example: This rule will not extract content of any drive items (files in document libraries) that haven't been modified for 60 days or more. [discrete#es-connectors-sharepoint-online-sync-rules-limitations] -======= Limitations of sync rules with incremental syncs +*Limitations of sync rules with incremental syncs* Changing sync rules after Sharepoint Online content has already been indexed can bring unexpected results, when using <>. @@ -288,7 +288,7 @@ Incremental syncs ensure _updates_ from 3rd-party system, but do not modify exis Let's take a look at several examples where incremental syncs might lead to inconsistent data on your index. [discrete#es-connectors-sharepoint-online-sync-rules-limitations-restrictive-added] -======== Example: Restrictive basic sync rule added after a full sync +*Example: Restrictive basic sync rule added after a full sync* Imagine your Sharepoint Online drive contains the following drive items: @@ -322,7 +322,7 @@ If no files were changed, incremental sync will not receive information about ch After a *full sync*, the index will be updated and files that are excluded by sync rules will be removed. [discrete#es-connectors-sharepoint-online-sync-rules-limitations-restrictive-removed] -======== Example: Restrictive basic sync rules removed after a full sync +*Example: Restrictive basic sync rules removed after a full sync* Imagine that Sharepoint Online drive has the following drive items: @@ -354,7 +354,7 @@ Afterwards, we can remove the filtering rule and run an incremental sync. If no Only a *full sync* will include the items previously ignored by the sync rule. [discrete#es-connectors-sharepoint-online-sync-rules-limitations-restrictive-changed] -======== Example: Advanced sync rules edge case +*Example: Advanced sync rules edge case* Advanced sync rules can be applied to limit which documents will have content extracted. For example, it's possible to set a rule so that documents older than 180 days won't have content extracted. @@ -763,7 +763,7 @@ Example: This rule will not extract content of any drive items (files in document libraries) that haven't been modified for 60 days or more. [discrete#es-connectors-sharepoint-online-client-sync-rules-limitations] -======= Limitations of sync rules with incremental syncs +*Limitations of sync rules with incremental syncs* Changing sync rules after Sharepoint Online content has already been indexed can bring unexpected results, when using <>. @@ -774,7 +774,7 @@ Incremental syncs ensure _updates_ from 3rd-party system, but do not modify exis Let's take a look at several examples where incremental syncs might lead to inconsistent data on your index. [discrete#es-connectors-sharepoint-online-client-sync-rules-limitations-restrictive-added] -======== Example: Restrictive basic sync rule added after a full sync +*Example: Restrictive basic sync rule added after a full sync* Imagine your Sharepoint Online drive contains the following drive items: @@ -808,7 +808,7 @@ If no files were changed, incremental sync will not receive information about ch After a *full sync*, the index will be updated and files that are excluded by sync rules will be removed. [discrete#es-connectors-sharepoint-online-client-sync-rules-limitations-restrictive-removed] -======== Example: Restrictive basic sync rules removed after a full sync +*Example: Restrictive basic sync rules removed after a full sync* Imagine that Sharepoint Online drive has the following drive items: @@ -840,7 +840,7 @@ Afterwards, we can remove the filtering rule and run an incremental sync. If no Only a *full sync* will include the items previously ignored by the sync rule. [discrete#es-connectors-sharepoint-online-client-sync-rules-limitations-restrictive-changed] -======== Example: Advanced sync rules edge case +*Example: Advanced sync rules edge case* Advanced sync rules can be applied to limit which documents will have content extracted. For example, it's possible to set a rule so that documents older than 180 days won't have content extracted. From b4e852a54be436d8b8036da0a0ec4a472d44524a Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Mon, 9 Dec 2024 12:00:12 +0100 Subject: [PATCH 119/204] [TEST] Wait for no pending operations on the index shard (#118244) This fixes testRetryPointInTime which on teardown is looking to assert that the operations in the translog and in the lucene index are the same. Previously we didn't wait for the translog operations to be applied. This changes `assertConsistentHistoryInLuceneIndex` to wait for the pending operations in the translog to be applied. Fixes #117116 --- muted-tests.yml | 3 --- .../elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 4523db7239be6..dcaa415a67966 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -157,9 +157,6 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=snapshot/10_basic/Create a source only snapshot and then restore it} issue: https://github.com/elastic/elasticsearch/issues/117295 -- class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests - method: testRetryPointInTime - issue: https://github.com/elastic/elasticsearch/issues/117116 - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT method: testInferDeploysDefaultElser issue: https://github.com/elastic/elasticsearch/issues/114913 diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 8bc81fef2157d..a2bf70bf6e087 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -128,6 +128,7 @@ protected Collection> nodePlugins() { @After public void assertConsistentHistoryInLuceneIndex() throws Exception { + internalCluster().beforeIndexDeletion(); internalCluster().assertConsistentHistoryBetweenTranslogAndLuceneIndex(); } From 67ee03411bad5beb7e8000f0b3fcbc8accdf96da Mon Sep 17 00:00:00 2001 From: kanoshiou <73424326+kanoshiou@users.noreply.github.com> Date: Mon, 9 Dec 2024 20:08:48 +0800 Subject: [PATCH 120/204] ESQL: Enable async get to support formatting (#111104) I've updated the listener for GET /_query/async/{id} to EsqlResponseListener, so it now accepts parameters (delimiter, drop_null_columns and format) like the POST /_query API. Additionally, I have added tests to verify the correctness of the code. You can now set the format in the request parameters to specify the return style. Closes #110926 --- docs/changelog/111104.yaml | 6 + .../esql/esql-async-query-get-api.asciidoc | 4 + .../async/AsyncTaskManagementService.java | 2 +- .../xpack/esql/qa/rest/RestEsqlTestCase.java | 203 ++++++++++++++---- .../esql/action/EsqlResponseListener.java | 66 +++--- .../action/RestEsqlGetAsyncResultAction.java | 3 +- .../esql/plugin/EsqlMediaTypeParser.java | 13 +- .../esql/plugin/EsqlMediaTypeParserTests.java | 13 +- 8 files changed, 236 insertions(+), 74 deletions(-) create mode 100644 docs/changelog/111104.yaml diff --git a/docs/changelog/111104.yaml b/docs/changelog/111104.yaml new file mode 100644 index 0000000000000..a7dffdd0be221 --- /dev/null +++ b/docs/changelog/111104.yaml @@ -0,0 +1,6 @@ +pr: 111104 +summary: "ESQL: Enable async get to support formatting" +area: ES|QL +type: feature +issues: + - 110926 diff --git a/docs/reference/esql/esql-async-query-get-api.asciidoc b/docs/reference/esql/esql-async-query-get-api.asciidoc index ec68313b2c490..82a6ae5b28b51 100644 --- a/docs/reference/esql/esql-async-query-get-api.asciidoc +++ b/docs/reference/esql/esql-async-query-get-api.asciidoc @@ -39,6 +39,10 @@ parameter is `true`. [[esql-async-query-get-api-query-params]] ==== {api-query-parms-title} +The API accepts the same parameters as the synchronous +<>, along with the following +parameters: + `wait_for_completion_timeout`:: (Optional, <>) Timeout duration to wait for the request to finish. Defaults to no timeout, diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/async/AsyncTaskManagementService.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/async/AsyncTaskManagementService.java index 94bac95b91501..91fdb9c39b6e3 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/async/AsyncTaskManagementService.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/async/AsyncTaskManagementService.java @@ -208,7 +208,7 @@ private ActionListener wrapStoringListener( ActionListener listener ) { AtomicReference> exclusiveListener = new AtomicReference<>(listener); - // This is will performed in case of timeout + // This will be performed in case of timeout Scheduler.ScheduledCancellable timeoutHandler = threadPool.schedule(() -> { ActionListener acquiredListener = exclusiveListener.getAndSet(null); if (acquiredListener != null) { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 505ab3adc553b..6a8779eef4efc 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -350,21 +350,21 @@ public void testTextMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); var builder = requestObjectBuilder().query(fromIndex() + " | keep keyword, integer | sort integer asc | limit 100"); - assertEquals(expectedTextBody("txt", count, null), runEsqlAsTextWithFormat(builder, "txt", null)); + assertEquals(expectedTextBody("txt", count, null), runEsqlAsTextWithFormat(builder, "txt", null, mode)); } public void testCSVMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); var builder = requestObjectBuilder().query(fromIndex() + " | keep keyword, integer | sort integer asc | limit 100"); - assertEquals(expectedTextBody("csv", count, '|'), runEsqlAsTextWithFormat(builder, "csv", '|')); + assertEquals(expectedTextBody("csv", count, '|'), runEsqlAsTextWithFormat(builder, "csv", '|', mode)); } public void testTSVMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); var builder = requestObjectBuilder().query(fromIndex() + " | keep keyword, integer | sort integer asc | limit 100"); - assertEquals(expectedTextBody("tsv", count, null), runEsqlAsTextWithFormat(builder, "tsv", null)); + assertEquals(expectedTextBody("tsv", count, null), runEsqlAsTextWithFormat(builder, "tsv", null, mode)); } public void testCSVNoHeaderMode() throws IOException { @@ -1003,53 +1003,35 @@ public static Map runEsqlSync(RequestObjectBuilder requestObject } public static Map runEsqlAsync(RequestObjectBuilder requestObject) throws IOException { - return runEsqlAsync(requestObject, new AssertWarnings.NoWarnings()); + return runEsqlAsync(requestObject, randomBoolean(), new AssertWarnings.NoWarnings()); } static Map runEsql(RequestObjectBuilder requestObject, AssertWarnings assertWarnings, Mode mode) throws IOException { if (mode == ASYNC) { - return runEsqlAsync(requestObject, assertWarnings); + return runEsqlAsync(requestObject, randomBoolean(), assertWarnings); } else { return runEsqlSync(requestObject, assertWarnings); } } public static Map runEsqlSync(RequestObjectBuilder requestObject, AssertWarnings assertWarnings) throws IOException { - requestObject.build(); - Request request = prepareRequest(SYNC); - String mediaType = attachBody(requestObject, request); - - RequestOptions.Builder options = request.getOptions().toBuilder(); - options.setWarningsHandler(WarningsHandler.PERMISSIVE); // We assert the warnings ourselves - options.addHeader("Content-Type", mediaType); - - if (randomBoolean()) { - options.addHeader("Accept", mediaType); - } else { - request.addParameter("format", requestObject.contentType().queryParameter()); - } - request.setOptions(options); + Request request = prepareRequestWithOptions(requestObject, SYNC); HttpEntity entity = performRequest(request, assertWarnings); return entityToMap(entity, requestObject.contentType()); } public static Map runEsqlAsync(RequestObjectBuilder requestObject, AssertWarnings assertWarnings) throws IOException { - addAsyncParameters(requestObject); - requestObject.build(); - Request request = prepareRequest(ASYNC); - String mediaType = attachBody(requestObject, request); - - RequestOptions.Builder options = request.getOptions().toBuilder(); - options.setWarningsHandler(WarningsHandler.PERMISSIVE); // We assert the warnings ourselves - options.addHeader("Content-Type", mediaType); + return runEsqlAsync(requestObject, randomBoolean(), assertWarnings); + } - if (randomBoolean()) { - options.addHeader("Accept", mediaType); - } else { - request.addParameter("format", requestObject.contentType().queryParameter()); - } - request.setOptions(options); + public static Map runEsqlAsync( + RequestObjectBuilder requestObject, + boolean keepOnCompletion, + AssertWarnings assertWarnings + ) throws IOException { + addAsyncParameters(requestObject, keepOnCompletion); + Request request = prepareRequestWithOptions(requestObject, ASYNC); if (shouldLog()) { LOGGER.info("REQUEST={}", request); @@ -1061,7 +1043,7 @@ public static Map runEsqlAsync(RequestObjectBuilder requestObjec Object initialColumns = null; Object initialValues = null; var json = entityToMap(entity, requestObject.contentType()); - checkKeepOnCompletion(requestObject, json); + checkKeepOnCompletion(requestObject, json, keepOnCompletion); String id = (String) json.get("id"); var supportsAsyncHeaders = clusterHasCapability("POST", "/_query", List.of(), List.of("async_query_status_headers")).orElse(false); @@ -1101,7 +1083,7 @@ public static Map runEsqlAsync(RequestObjectBuilder requestObjec // issue a second request to "async get" the results Request getRequest = prepareAsyncGetRequest(id); - getRequest.setOptions(options); + getRequest.setOptions(request.getOptions()); response = performRequest(getRequest); entity = response.getEntity(); } @@ -1119,6 +1101,66 @@ public static Map runEsqlAsync(RequestObjectBuilder requestObjec return removeAsyncProperties(result); } + public void testAsyncGetWithoutContentType() throws IOException { + int count = randomIntBetween(0, 100); + bulkLoadTestData(count); + var requestObject = requestObjectBuilder().query(fromIndex() + " | keep keyword, integer | sort integer asc | limit 100"); + + addAsyncParameters(requestObject, true); + Request request = prepareRequestWithOptions(requestObject, ASYNC); + + if (shouldLog()) { + LOGGER.info("REQUEST={}", request); + } + + Response response = performRequest(request); + HttpEntity entity = response.getEntity(); + + var json = entityToMap(entity, requestObject.contentType()); + checkKeepOnCompletion(requestObject, json, true); + String id = (String) json.get("id"); + // results won't be returned since keepOnCompletion is true + assertThat(id, is(not(emptyOrNullString()))); + + // issue an "async get" request with no Content-Type + Request getRequest = prepareAsyncGetRequest(id); + response = performRequest(getRequest); + entity = response.getEntity(); + var result = entityToMap(entity, XContentType.JSON); + + ListMatcher values = matchesList(); + for (int i = 0; i < count; i++) { + values = values.item(matchesList().item("keyword" + i).item(i)); + } + assertMap( + result, + matchesMap().entry( + "columns", + matchesList().item(matchesMap().entry("name", "keyword").entry("type", "keyword")) + .item(matchesMap().entry("name", "integer").entry("type", "integer")) + ).entry("values", values).entry("took", greaterThanOrEqualTo(0)).entry("id", id).entry("is_running", false) + ); + + } + + static Request prepareRequestWithOptions(RequestObjectBuilder requestObject, Mode mode) throws IOException { + requestObject.build(); + Request request = prepareRequest(mode); + String mediaType = attachBody(requestObject, request); + + RequestOptions.Builder options = request.getOptions().toBuilder(); + options.setWarningsHandler(WarningsHandler.PERMISSIVE); // We assert the warnings ourselves + options.addHeader("Content-Type", mediaType); + + if (randomBoolean()) { + options.addHeader("Accept", mediaType); + } else { + request.addParameter("format", requestObject.contentType().queryParameter()); + } + request.setOptions(options); + return request; + } + // Removes async properties, otherwise consuming assertions would need to handle sync and async differences static Map removeAsyncProperties(Map map) { Map copy = new HashMap<>(map); @@ -1139,17 +1181,20 @@ protected static Map entityToMap(HttpEntity entity, XContentType } } - static void addAsyncParameters(RequestObjectBuilder requestObject) throws IOException { + static void addAsyncParameters(RequestObjectBuilder requestObject, boolean keepOnCompletion) throws IOException { // deliberately short in order to frequently trigger return without results requestObject.waitForCompletion(TimeValue.timeValueNanos(randomIntBetween(1, 100))); - requestObject.keepOnCompletion(randomBoolean()); + requestObject.keepOnCompletion(keepOnCompletion); requestObject.keepAlive(TimeValue.timeValueDays(randomIntBetween(1, 10))); } // If keep_on_completion is set then an id must always be present, regardless of the value of any other property. - static void checkKeepOnCompletion(RequestObjectBuilder requestObject, Map json) { + static void checkKeepOnCompletion(RequestObjectBuilder requestObject, Map json, boolean keepOnCompletion) { if (requestObject.keepOnCompletion()) { + assertTrue(keepOnCompletion); assertThat((String) json.get("id"), not(emptyOrNullString())); + } else { + assertFalse(keepOnCompletion); } } @@ -1167,14 +1212,19 @@ static void deleteNonExistent(Request request) throws IOException { assertEquals(404, response.getStatusLine().getStatusCode()); } - static String runEsqlAsTextWithFormat(RequestObjectBuilder builder, String format, @Nullable Character delimiter) throws IOException { - Request request = prepareRequest(SYNC); + static String runEsqlAsTextWithFormat(RequestObjectBuilder builder, String format, @Nullable Character delimiter, Mode mode) + throws IOException { + Request request = prepareRequest(mode); + if (mode == ASYNC) { + addAsyncParameters(builder, randomBoolean()); + } String mediaType = attachBody(builder.build(), request); RequestOptions.Builder options = request.getOptions().toBuilder(); options.addHeader("Content-Type", mediaType); - if (randomBoolean()) { + boolean addParam = randomBoolean(); + if (addParam) { request.addParameter("format", format); } else { switch (format) { @@ -1188,8 +1238,75 @@ static String runEsqlAsTextWithFormat(RequestObjectBuilder builder, String forma } request.setOptions(options); - HttpEntity entity = performRequest(request, new AssertWarnings.NoWarnings()); - return Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)); + if (shouldLog()) { + LOGGER.info("REQUEST={}", request); + } + + Response response = performRequest(request); + HttpEntity entity = assertWarnings(response, new AssertWarnings.NoWarnings()); + + // get the content, it could be empty because the request might have not completed + String initialValue = Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)); + String id = response.getHeader("X-Elasticsearch-Async-Id"); + + if (mode == SYNC) { + assertThat(id, is(emptyOrNullString())); + return initialValue; + } + + if (id == null) { + // no id returned from an async call, must have completed immediately and without keep_on_completion + assertThat(builder.keepOnCompletion(), either(nullValue()).or(is(false))); + assertNull(response.getHeader("is_running")); + // the content cant be empty + assertThat(initialValue, not(emptyOrNullString())); + return initialValue; + } else { + // async may not return results immediately, so may need an async get + assertThat(id, is(not(emptyOrNullString()))); + String isRunning = response.getHeader("X-Elasticsearch-Async-Is-Running"); + if ("?0".equals(isRunning)) { + // must have completed immediately so keep_on_completion must be true + assertThat(builder.keepOnCompletion(), is(true)); + } else { + // did not return results immediately, so we will need an async get + // Also, different format modes return different results. + switch (format) { + case "txt" -> assertThat(initialValue, emptyOrNullString()); + case "csv" -> { + assertEquals(initialValue, "\r\n"); + initialValue = ""; + } + case "tsv" -> { + assertEquals(initialValue, "\n"); + initialValue = ""; + } + } + } + // issue a second request to "async get" the results + Request getRequest = prepareAsyncGetRequest(id); + if (delimiter != null) { + getRequest.addParameter("delimiter", String.valueOf(delimiter)); + } + // If the `format` parameter is not added, the GET request will return a response + // with the `Content-Type` type due to the lack of an `Accept` header. + if (addParam) { + getRequest.addParameter("format", format); + } + // if `addParam` is false, `options` will already have an `Accept` header + getRequest.setOptions(options); + response = performRequest(getRequest); + entity = assertWarnings(response, new AssertWarnings.NoWarnings()); + } + String newValue = Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)); + + // assert initial contents, if any, are the same as async get contents + if (initialValue != null && initialValue.isEmpty() == false) { + assertEquals(initialValue, newValue); + } + + assertDeletable(id); + return newValue; } private static Request prepareRequest(Mode mode) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java index 1c88fe6f45d81..fb7e0f651458c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java @@ -22,6 +22,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; import org.elasticsearch.xcontent.MediaType; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.esql.arrow.ArrowFormat; import org.elasticsearch.xpack.esql.arrow.ArrowResponse; import org.elasticsearch.xpack.esql.formatter.TextFormat; @@ -87,7 +88,7 @@ public TimeValue stop() { /** * Keep the initial query for logging purposes. */ - private final String esqlQuery; + private final String esqlQueryOrId; /** * Stop the time it took to build a response to later log it. Use something thread-safe here because stopping time requires state and * {@link EsqlResponseListener} might be used from different threads. @@ -98,29 +99,23 @@ public TimeValue stop() { * To correctly time the execution of a request, a {@link EsqlResponseListener} must be constructed immediately before execution begins. */ public EsqlResponseListener(RestChannel channel, RestRequest restRequest, EsqlQueryRequest esqlRequest) { - super(channel); + this(channel, restRequest, esqlRequest.query(), EsqlMediaTypeParser.getResponseMediaType(restRequest, esqlRequest)); + } + /** + * Async query GET API does not have an EsqlQueryRequest. + */ + public EsqlResponseListener(RestChannel channel, RestRequest getRequest) { + this(channel, getRequest, getRequest.param("id"), EsqlMediaTypeParser.getResponseMediaType(getRequest, XContentType.JSON)); + } + + private EsqlResponseListener(RestChannel channel, RestRequest restRequest, String esqlQueryOrId, MediaType mediaType) { + super(channel); this.channel = channel; this.restRequest = restRequest; - this.esqlQuery = esqlRequest.query(); - mediaType = EsqlMediaTypeParser.getResponseMediaType(restRequest, esqlRequest); - - /* - * Special handling for the "delimiter" parameter which should only be - * checked for being present or not in the case of CSV format. We cannot - * override {@link BaseRestHandler#responseParams()} because this - * parameter should only be checked for CSV, not other formats. - */ - if (mediaType != CSV && restRequest.hasParam(URL_PARAM_DELIMITER)) { - String message = String.format( - Locale.ROOT, - "parameter: [%s] can only be used with the format [%s] for request [%s]", - URL_PARAM_DELIMITER, - CSV.queryParameter(), - restRequest.path() - ); - throw new IllegalArgumentException(message); - } + this.esqlQueryOrId = esqlQueryOrId; + this.mediaType = mediaType; + checkDelimiter(); } @Override @@ -197,14 +192,18 @@ public ActionListener wrapWithLogging() { listener.onResponse(r); // At this point, the StopWatch should already have been stopped, so we log a consistent time. LOGGER.debug( - "Finished execution of ESQL query.\nQuery string: [{}]\nExecution time: [{}]ms", - esqlQuery, + "Finished execution of ESQL query.\nQuery string or async ID: [{}]\nExecution time: [{}]ms", + esqlQueryOrId, getTook(r, TimeUnit.MILLISECONDS) ); }, ex -> { // In case of failure, stop the time manually before sending out the response. long timeMillis = getTook(null, TimeUnit.MILLISECONDS); - LOGGER.debug("Failed execution of ESQL query.\nQuery string: [{}]\nExecution time: [{}]ms", esqlQuery, timeMillis); + LOGGER.debug( + "Failed execution of ESQL query.\nQuery string or async ID: [{}]\nExecution time: [{}]ms", + esqlQueryOrId, + timeMillis + ); listener.onFailure(ex); }); } @@ -213,4 +212,23 @@ static void logOnFailure(Throwable throwable) { RestStatus status = ExceptionsHelper.status(throwable); LOGGER.log(status.getStatus() >= 500 ? Level.WARN : Level.DEBUG, () -> "Request failed with status [" + status + "]: ", throwable); } + + /* + * Special handling for the "delimiter" parameter which should only be + * checked for being present or not in the case of CSV format. We cannot + * override {@link BaseRestHandler#responseParams()} because this + * parameter should only be checked for CSV, not other formats. + */ + private void checkDelimiter() { + if (mediaType != CSV && restRequest.hasParam(URL_PARAM_DELIMITER)) { + String message = String.format( + Locale.ROOT, + "parameter: [%s] can only be used with the format [%s] for request [%s]", + URL_PARAM_DELIMITER, + CSV.queryParameter(), + restRequest.path() + ); + throw new IllegalArgumentException(message); + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlGetAsyncResultAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlGetAsyncResultAction.java index b5a1821350e5e..848a75d7fb19f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlGetAsyncResultAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlGetAsyncResultAction.java @@ -12,7 +12,6 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; -import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; import java.util.List; @@ -43,7 +42,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli if (request.hasParam("keep_alive")) { get.setKeepAlive(request.paramAsTime("keep_alive", get.getKeepAlive())); } - return channel -> client.execute(EsqlAsyncGetResultAction.INSTANCE, get, new RestRefCountedChunkedToXContentListener<>(channel)); + return channel -> client.execute(EsqlAsyncGetResultAction.INSTANCE, get, new EsqlResponseListener(channel, request)); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java index 17329ca2e0054..1931692cea8bc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java @@ -42,16 +42,23 @@ public class EsqlMediaTypeParser { * combinations are detected. */ public static MediaType getResponseMediaType(RestRequest request, EsqlQueryRequest esqlRequest) { - var mediaType = request.hasParam(URL_PARAM_FORMAT) ? mediaTypeFromParams(request) : mediaTypeFromHeaders(request); + var mediaType = getResponseMediaType(request, (MediaType) null); validateColumnarRequest(esqlRequest.columnar(), mediaType); validateIncludeCCSMetadata(esqlRequest.includeCCSMetadata(), mediaType); return checkNonNullMediaType(mediaType, request); } + /* + * Retrieve the mediaType of a REST request. If no mediaType can be established from the request, return the provided default. + */ + public static MediaType getResponseMediaType(RestRequest request, MediaType defaultMediaType) { + var mediaType = request.hasParam(URL_PARAM_FORMAT) ? mediaTypeFromParams(request) : mediaTypeFromHeaders(request); + return mediaType == null ? defaultMediaType : mediaType; + } + private static MediaType mediaTypeFromHeaders(RestRequest request) { ParsedMediaType acceptType = request.getParsedAccept(); - MediaType mediaType = acceptType != null ? acceptType.toMediaType(MEDIA_TYPE_REGISTRY) : request.getXContentType(); - return checkNonNullMediaType(mediaType, request); + return acceptType != null ? acceptType.toMediaType(MEDIA_TYPE_REGISTRY) : request.getXContentType(); } private static MediaType mediaTypeFromParams(RestRequest request) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java index 4b9166c621940..4758f83c42bb7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java @@ -17,6 +17,7 @@ import java.util.Collections; import java.util.Map; +import static org.elasticsearch.xcontent.XContentType.JSON; import static org.elasticsearch.xpack.esql.formatter.TextFormat.CSV; import static org.elasticsearch.xpack.esql.formatter.TextFormat.PLAIN_TEXT; import static org.elasticsearch.xpack.esql.formatter.TextFormat.TSV; @@ -123,11 +124,17 @@ public void testIncludeCCSMetadataWithNonJSONMediaTypesInParams() { public void testNoFormat() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> getResponseMediaType(new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).build(), createTestInstance(false)) + () -> getResponseMediaType(emptyRequest(), createTestInstance(false)) ); assertEquals(e.getMessage(), "Invalid request content type: Accept=[null], Content-Type=[null], format=[null]"); } + public void testNoContentType() { + RestRequest fakeRestRequest = emptyRequest(); + assertThat(getResponseMediaType(fakeRestRequest, CSV), is(CSV)); + assertThat(getResponseMediaType(fakeRestRequest, JSON), is(JSON)); + } + private static RestRequest reqWithAccept(String acceptHeader) { return new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withHeaders( Map.of("Content-Type", Collections.singletonList("application/json"), "Accept", Collections.singletonList(acceptHeader)) @@ -140,6 +147,10 @@ private static RestRequest reqWithParams(Map params) { ).withParams(params).build(); } + private static RestRequest emptyRequest() { + return new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).build(); + } + protected EsqlQueryRequest createTestInstance(boolean columnar) { var request = new EsqlQueryRequest(); request.columnar(columnar); From 64e0902f58a350b666708a85c9506dccdd7ecc0b Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 9 Dec 2024 12:11:55 +0000 Subject: [PATCH 121/204] Pull AWS SDK versions to top level (#118247) Today each relevant module defines the version of the AWS SDK that it uses, which means there's a risk that we use different versions in different modules. This commit pulls the version declarations to the top level to make sure we keep everything in sync. --- build-tools-internal/version.properties | 2 + modules/repository-s3/build.gradle | 12 ++---- plugins/discovery-ec2/build.gradle | 8 +--- x-pack/plugin/inference/build.gradle | 54 ++++++++++++------------- 4 files changed, 33 insertions(+), 43 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 29c5bc16a8c4a..aaf654a37dd22 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -17,6 +17,8 @@ jna = 5.12.1 netty = 4.1.115.Final commons_lang3 = 3.9 google_oauth_client = 1.34.1 +awsv1sdk = 1.12.270 +awsv2sdk = 2.28.13 antlr4 = 4.13.1 # bouncy castle version for non-fips. fips jars use a different version diff --git a/modules/repository-s3/build.gradle b/modules/repository-s3/build.gradle index 2cfb5d23db4ff..f0dc1ca714958 100644 --- a/modules/repository-s3/build.gradle +++ b/modules/repository-s3/build.gradle @@ -18,15 +18,11 @@ esplugin { classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin' } -versions << [ - 'aws': '1.12.270' -] - dependencies { - api "com.amazonaws:aws-java-sdk-s3:${versions.aws}" - api "com.amazonaws:aws-java-sdk-core:${versions.aws}" - api "com.amazonaws:aws-java-sdk-sts:${versions.aws}" - api "com.amazonaws:jmespath-java:${versions.aws}" + api "com.amazonaws:aws-java-sdk-s3:${versions.awsv1sdk}" + api "com.amazonaws:aws-java-sdk-core:${versions.awsv1sdk}" + api "com.amazonaws:aws-java-sdk-sts:${versions.awsv1sdk}" + api "com.amazonaws:jmespath-java:${versions.awsv1sdk}" api "org.apache.httpcomponents:httpclient:${versions.httpclient}" api "org.apache.httpcomponents:httpcore:${versions.httpcore}" api "commons-logging:commons-logging:${versions.commonslogging}" diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index 980e2467206d7..a4321a2d61f98 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -14,13 +14,9 @@ esplugin { classname 'org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin' } -versions << [ - 'aws': '1.12.270' -] - dependencies { - api "com.amazonaws:aws-java-sdk-ec2:${versions.aws}" - api "com.amazonaws:aws-java-sdk-core:${versions.aws}" + api "com.amazonaws:aws-java-sdk-ec2:${versions.awsv1sdk}" + api "com.amazonaws:aws-java-sdk-core:${versions.awsv1sdk}" api "org.apache.httpcomponents:httpclient:${versions.httpclient}" api "org.apache.httpcomponents:httpcore:${versions.httpcore}" api "commons-logging:commons-logging:${versions.commonslogging}" diff --git a/x-pack/plugin/inference/build.gradle b/x-pack/plugin/inference/build.gradle index 3c19e11a450b4..1d0236a5834e5 100644 --- a/x-pack/plugin/inference/build.gradle +++ b/x-pack/plugin/inference/build.gradle @@ -26,10 +26,6 @@ base { archivesName = 'x-pack-inference' } -versions << [ - 'aws2': '2.28.13' -] - dependencies { implementation project(path: ':libs:logging') compileOnly project(":server") @@ -62,36 +58,36 @@ dependencies { implementation 'io.opencensus:opencensus-contrib-http-util:0.31.1' /* AWS SDK v2 */ - implementation ("software.amazon.awssdk:bedrockruntime:${versions.aws2}") - api "software.amazon.awssdk:protocol-core:${versions.aws2}" - api "software.amazon.awssdk:aws-json-protocol:${versions.aws2}" - api "software.amazon.awssdk:third-party-jackson-core:${versions.aws2}" - api "software.amazon.awssdk:http-auth-aws:${versions.aws2}" - api "software.amazon.awssdk:checksums-spi:${versions.aws2}" - api "software.amazon.awssdk:checksums:${versions.aws2}" - api "software.amazon.awssdk:sdk-core:${versions.aws2}" + implementation ("software.amazon.awssdk:bedrockruntime:${versions.awsv2sdk}") + api "software.amazon.awssdk:protocol-core:${versions.awsv2sdk}" + api "software.amazon.awssdk:aws-json-protocol:${versions.awsv2sdk}" + api "software.amazon.awssdk:third-party-jackson-core:${versions.awsv2sdk}" + api "software.amazon.awssdk:http-auth-aws:${versions.awsv2sdk}" + api "software.amazon.awssdk:checksums-spi:${versions.awsv2sdk}" + api "software.amazon.awssdk:checksums:${versions.awsv2sdk}" + api "software.amazon.awssdk:sdk-core:${versions.awsv2sdk}" api "org.reactivestreams:reactive-streams:1.0.4" api "org.reactivestreams:reactive-streams-tck:1.0.4" - api "software.amazon.awssdk:profiles:${versions.aws2}" - api "software.amazon.awssdk:retries:${versions.aws2}" - api "software.amazon.awssdk:auth:${versions.aws2}" - api "software.amazon.awssdk:http-auth-aws-eventstream:${versions.aws2}" + api "software.amazon.awssdk:profiles:${versions.awsv2sdk}" + api "software.amazon.awssdk:retries:${versions.awsv2sdk}" + api "software.amazon.awssdk:auth:${versions.awsv2sdk}" + api "software.amazon.awssdk:http-auth-aws-eventstream:${versions.awsv2sdk}" api "software.amazon.eventstream:eventstream:1.0.1" - api "software.amazon.awssdk:http-auth-spi:${versions.aws2}" - api "software.amazon.awssdk:http-auth:${versions.aws2}" - api "software.amazon.awssdk:identity-spi:${versions.aws2}" - api "software.amazon.awssdk:http-client-spi:${versions.aws2}" - api "software.amazon.awssdk:regions:${versions.aws2}" - api "software.amazon.awssdk:annotations:${versions.aws2}" - api "software.amazon.awssdk:utils:${versions.aws2}" - api "software.amazon.awssdk:aws-core:${versions.aws2}" - api "software.amazon.awssdk:metrics-spi:${versions.aws2}" - api "software.amazon.awssdk:json-utils:${versions.aws2}" - api "software.amazon.awssdk:endpoints-spi:${versions.aws2}" - api "software.amazon.awssdk:retries-spi:${versions.aws2}" + api "software.amazon.awssdk:http-auth-spi:${versions.awsv2sdk}" + api "software.amazon.awssdk:http-auth:${versions.awsv2sdk}" + api "software.amazon.awssdk:identity-spi:${versions.awsv2sdk}" + api "software.amazon.awssdk:http-client-spi:${versions.awsv2sdk}" + api "software.amazon.awssdk:regions:${versions.awsv2sdk}" + api "software.amazon.awssdk:annotations:${versions.awsv2sdk}" + api "software.amazon.awssdk:utils:${versions.awsv2sdk}" + api "software.amazon.awssdk:aws-core:${versions.awsv2sdk}" + api "software.amazon.awssdk:metrics-spi:${versions.awsv2sdk}" + api "software.amazon.awssdk:json-utils:${versions.awsv2sdk}" + api "software.amazon.awssdk:endpoints-spi:${versions.awsv2sdk}" + api "software.amazon.awssdk:retries-spi:${versions.awsv2sdk}" /* Netty (via AWS SDKv2) */ - implementation "software.amazon.awssdk:netty-nio-client:${versions.aws2}" + implementation "software.amazon.awssdk:netty-nio-client:${versions.awsv2sdk}" runtimeOnly "io.netty:netty-buffer:${versions.netty}" runtimeOnly "io.netty:netty-codec-dns:${versions.netty}" runtimeOnly "io.netty:netty-codec-http2:${versions.netty}" From 63ee866ed6f7e27ddd3364660c5606ea20ab73e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= Date: Mon, 9 Dec 2024 14:05:48 +0100 Subject: [PATCH 122/204] ESQL: Categorize grouping function testing improvements (#118013) Added some extra tests on the CategorizeBlockHash. Added NullFold rule comments, and forced nullable() to TRUE on Categorize. --- .../esql/core/expression/Nullability.java | 17 +- .../blockhash/CategorizeBlockHashTests.java | 235 ++++++++++++++---- .../src/main/resources/categorize.csv-spec | 23 ++ .../function/grouping/Categorize.java | 7 + .../optimizer/rules/logical/FoldNull.java | 5 +- 5 files changed, 229 insertions(+), 58 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Nullability.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Nullability.java index b08024a707774..d9f136a357208 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Nullability.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Nullability.java @@ -7,7 +7,18 @@ package org.elasticsearch.xpack.esql.core.expression; public enum Nullability { - TRUE, // Whether the expression can become null - FALSE, // The expression can never become null - UNKNOWN // Cannot determine if the expression supports possible null folding + /** + * Whether the expression can become null + */ + TRUE, + + /** + * The expression can never become null + */ + FALSE, + + /** + * Cannot determine if the expression supports possible null folding + */ + UNKNOWN } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/CategorizeBlockHashTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/CategorizeBlockHashTests.java index 3c47e85a4a9c8..f8428b7c33568 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/CategorizeBlockHashTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/CategorizeBlockHashTests.java @@ -50,11 +50,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.elasticsearch.compute.operator.OperatorTestCase.runDriver; +import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -95,41 +95,114 @@ public void testCategorizeRaw() { page = new Page(builder.build()); } - try (BlockHash hash = new CategorizeBlockHash(blockFactory, 0, AggregatorMode.INITIAL, analysisRegistry)) { - hash.add(page, new GroupingAggregatorFunction.AddInput() { - @Override - public void add(int positionOffset, IntBlock groupIds) { - assertEquals(groupIds.getPositionCount(), positions); + try (var hash = new CategorizeBlockHash(blockFactory, 0, AggregatorMode.SINGLE, analysisRegistry)) { + for (int i = randomInt(2); i < 3; i++) { + hash.add(page, new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + assertEquals(groupIds.getPositionCount(), positions); + + assertEquals(1, groupIds.getInt(0)); + assertEquals(2, groupIds.getInt(1)); + assertEquals(2, groupIds.getInt(2)); + assertEquals(2, groupIds.getInt(3)); + assertEquals(3, groupIds.getInt(4)); + assertEquals(1, groupIds.getInt(5)); + assertEquals(1, groupIds.getInt(6)); + if (withNull) { + assertEquals(0, groupIds.getInt(7)); + } + } - assertEquals(1, groupIds.getInt(0)); - assertEquals(2, groupIds.getInt(1)); - assertEquals(2, groupIds.getInt(2)); - assertEquals(2, groupIds.getInt(3)); - assertEquals(3, groupIds.getInt(4)); - assertEquals(1, groupIds.getInt(5)); - assertEquals(1, groupIds.getInt(6)); - if (withNull) { - assertEquals(0, groupIds.getInt(7)); + @Override + public void add(int positionOffset, IntVector groupIds) { + add(positionOffset, groupIds.asBlock()); } - } - @Override - public void add(int positionOffset, IntVector groupIds) { - add(positionOffset, groupIds.asBlock()); - } + @Override + public void close() { + fail("hashes should not close AddInput"); + } + }); - @Override - public void close() { - fail("hashes should not close AddInput"); - } - }); + assertHashState(hash, withNull, ".*?Connected.+?to.*?", ".*?Connection.+?error.*?", ".*?Disconnected.*?"); + } } finally { page.releaseBlocks(); } - // TODO: randomize and try multiple pages. - // TODO: assert the state of the BlockHash after adding pages. Including the categorizer state. - // TODO: also test the lookup method and other stuff. + // TODO: randomize values? May give wrong results + // TODO: assert the categorizer state after adding pages. + } + + public void testCategorizeRawMultivalue() { + final Page page; + boolean withNull = randomBoolean(); + final int positions = 3 + (withNull ? 1 : 0); + try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(positions)) { + builder.beginPositionEntry(); + builder.appendBytesRef(new BytesRef("Connected to 10.1.0.1")); + builder.appendBytesRef(new BytesRef("Connection error")); + builder.appendBytesRef(new BytesRef("Connection error")); + builder.appendBytesRef(new BytesRef("Connection error")); + builder.endPositionEntry(); + builder.appendBytesRef(new BytesRef("Disconnected")); + builder.beginPositionEntry(); + builder.appendBytesRef(new BytesRef("Connected to 10.1.0.2")); + builder.appendBytesRef(new BytesRef("Connected to 10.1.0.3")); + builder.endPositionEntry(); + if (withNull) { + if (randomBoolean()) { + builder.appendNull(); + } else { + builder.appendBytesRef(new BytesRef("")); + } + } + page = new Page(builder.build()); + } + + try (var hash = new CategorizeBlockHash(blockFactory, 0, AggregatorMode.SINGLE, analysisRegistry)) { + for (int i = randomInt(2); i < 3; i++) { + hash.add(page, new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + assertEquals(groupIds.getPositionCount(), positions); + + assertThat(groupIds.getFirstValueIndex(0), equalTo(0)); + assertThat(groupIds.getValueCount(0), equalTo(4)); + assertThat(groupIds.getFirstValueIndex(1), equalTo(4)); + assertThat(groupIds.getValueCount(1), equalTo(1)); + assertThat(groupIds.getFirstValueIndex(2), equalTo(5)); + assertThat(groupIds.getValueCount(2), equalTo(2)); + + assertEquals(1, groupIds.getInt(0)); + assertEquals(2, groupIds.getInt(1)); + assertEquals(2, groupIds.getInt(2)); + assertEquals(2, groupIds.getInt(3)); + assertEquals(3, groupIds.getInt(4)); + assertEquals(1, groupIds.getInt(5)); + assertEquals(1, groupIds.getInt(6)); + if (withNull) { + assertEquals(0, groupIds.getInt(7)); + } + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + add(positionOffset, groupIds.asBlock()); + } + + @Override + public void close() { + fail("hashes should not close AddInput"); + } + }); + + assertHashState(hash, withNull, ".*?Connected.+?to.*?", ".*?Connection.+?error.*?", ".*?Disconnected.*?"); + } + } finally { + page.releaseBlocks(); + } } public void testCategorizeIntermediate() { @@ -226,18 +299,18 @@ public void close() { page2.releaseBlocks(); } - try (BlockHash intermediateHash = new CategorizeBlockHash(blockFactory, 0, AggregatorMode.INTERMEDIATE, null)) { + try (var intermediateHash = new CategorizeBlockHash(blockFactory, 0, AggregatorMode.FINAL, null)) { intermediateHash.add(intermediatePage1, new GroupingAggregatorFunction.AddInput() { @Override public void add(int positionOffset, IntBlock groupIds) { - Set values = IntStream.range(0, groupIds.getPositionCount()) + List values = IntStream.range(0, groupIds.getPositionCount()) .map(groupIds::getInt) .boxed() - .collect(Collectors.toSet()); + .collect(Collectors.toList()); if (withNull) { - assertEquals(Set.of(0, 1, 2), values); + assertEquals(List.of(0, 1, 2), values); } else { - assertEquals(Set.of(1, 2), values); + assertEquals(List.of(1, 2), values); } } @@ -252,28 +325,39 @@ public void close() { } }); - intermediateHash.add(intermediatePage2, new GroupingAggregatorFunction.AddInput() { - @Override - public void add(int positionOffset, IntBlock groupIds) { - Set values = IntStream.range(0, groupIds.getPositionCount()) - .map(groupIds::getInt) - .boxed() - .collect(Collectors.toSet()); - // The category IDs {0, 1, 2} should map to groups {0, 2, 3}, because - // 0 matches an existing category (Connected to ...), and the others are new. - assertEquals(Set.of(1, 3, 4), values); - } + for (int i = randomInt(2); i < 3; i++) { + intermediateHash.add(intermediatePage2, new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + List values = IntStream.range(0, groupIds.getPositionCount()) + .map(groupIds::getInt) + .boxed() + .collect(Collectors.toList()); + // The category IDs {1, 2, 3} should map to groups {1, 3, 4}, because + // 1 matches an existing category (Connected to ...), and the others are new. + assertEquals(List.of(3, 1, 4), values); + } - @Override - public void add(int positionOffset, IntVector groupIds) { - add(positionOffset, groupIds.asBlock()); - } + @Override + public void add(int positionOffset, IntVector groupIds) { + add(positionOffset, groupIds.asBlock()); + } - @Override - public void close() { - fail("hashes should not close AddInput"); - } - }); + @Override + public void close() { + fail("hashes should not close AddInput"); + } + }); + + assertHashState( + intermediateHash, + withNull, + ".*?Connected.+?to.*?", + ".*?Connection.+?error.*?", + ".*?Disconnected.*?", + ".*?System.+?shutdown.*?" + ); + } } finally { intermediatePage1.releaseBlocks(); intermediatePage2.releaseBlocks(); @@ -457,4 +541,49 @@ public void testCategorize_withDriver() { private BlockHash.GroupSpec makeGroupSpec() { return new BlockHash.GroupSpec(0, ElementType.BYTES_REF, true); } + + private void assertHashState(CategorizeBlockHash hash, boolean withNull, String... expectedKeys) { + // Check the keys + Block[] blocks = null; + try { + blocks = hash.getKeys(); + assertThat(blocks, arrayWithSize(1)); + + var keysBlock = (BytesRefBlock) blocks[0]; + assertThat(keysBlock.getPositionCount(), equalTo(expectedKeys.length + (withNull ? 1 : 0))); + + if (withNull) { + assertTrue(keysBlock.isNull(0)); + } + + for (int i = 0; i < expectedKeys.length; i++) { + int position = i + (withNull ? 1 : 0); + String key = keysBlock.getBytesRef(position, new BytesRef()).utf8ToString(); + assertThat(key, equalTo(expectedKeys[i])); + } + } finally { + if (blocks != null) { + Releasables.close(blocks); + } + } + + // Check the nonEmpty() result + try (IntVector nonEmptyKeys = hash.nonEmpty()) { + int oneIfNull = withNull ? 1 : 0; + assertThat(nonEmptyKeys.getPositionCount(), equalTo(expectedKeys.length + oneIfNull)); + + for (int i = 0; i < expectedKeys.length + oneIfNull; i++) { + assertThat(nonEmptyKeys.getInt(i), equalTo(i + 1 - oneIfNull)); + } + } + + // Check seenGroupIds() + try (var seenGroupIds = hash.seenGroupIds(blockFactory.bigArrays())) { + assertThat(seenGroupIds.get(0), equalTo(withNull)); + + for (int i = 1; i <= expectedKeys.length; i++) { + assertThat(seenGroupIds.get(i), equalTo(true)); + } + } + } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/categorize.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/categorize.csv-spec index 804c1c56a1eb5..4ce43961a7077 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/categorize.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/categorize.csv-spec @@ -374,6 +374,29 @@ COUNT():long | category:keyword 7 | null ; +on const null +required_capability: categorize_v5 + +FROM sample_data + | STATS COUNT(), SUM(event_duration) BY category=CATEGORIZE(null) + | SORT category +; + +COUNT():long | SUM(event_duration):long | category:keyword + 7 | 23231327 | null +; + +on null row +required_capability: categorize_v5 + +ROW message = null, str = ["a", "b", "c"] +| STATS COUNT(), VALUES(str) BY category=CATEGORIZE(message) +; + +COUNT():long | VALUES(str):keyword | category:keyword + 1 | [a, b, c] | null +; + filtering out all data required_capability: categorize_v5 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java index e2c04ecb15b59..ded913a78bdf1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java @@ -13,6 +13,7 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.capabilities.Validatable; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -92,6 +93,12 @@ public boolean foldable() { return false; } + @Override + public Nullability nullable() { + // Both nulls and empty strings result in null values + return Nullability.TRUE; + } + @Override public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { throw new UnsupportedOperationException("CATEGORIZE is only evaluated during aggregations"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java index 4f97bf60bd863..747864625e65c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java @@ -41,8 +41,9 @@ public Expression rule(Expression e) { if (Expressions.isGuaranteedNull(in.value())) { return Literal.of(in, null); } - } else if (e instanceof Alias == false - && e.nullable() == Nullability.TRUE + } else if (e instanceof Alias == false && e.nullable() == Nullability.TRUE + // Categorize function stays as a STATS grouping (It isn't moved to an early EVAL like other groupings), + // so folding it to null would currently break the plan, as we don't create an attribute/channel for that null value. && e instanceof Categorize == false && Expressions.anyMatch(e.children(), Expressions::isGuaranteedNull)) { return Literal.of(e, null); From ccc416ddf9e92a09865fdd765ec5eefb6d9456b2 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Mon, 9 Dec 2024 14:13:16 +0100 Subject: [PATCH 123/204] Ignore order in LookupMessageFromIndexKeepReordered (#118256) Fix #118150 Fix #118151 We should also ignore the order for this test, as the output rows are not deterministic in all cases. --- muted-tests.yml | 6 ------ .../qa/testFixtures/src/main/resources/lookup-join.csv-spec | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index dcaa415a67966..eecb7ac3d7e59 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -245,12 +245,6 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveTests method: test41AutoconfigurationNotTriggeredWhenNodeCannotContainData issue: https://github.com/elastic/elasticsearch/issues/118110 -- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT - method: test {lookup-join.LookupMessageFromIndexKeepReordered SYNC} - issue: https://github.com/elastic/elasticsearch/issues/118150 -- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT - method: test {lookup-join.LookupMessageFromIndexKeepReordered ASYNC} - issue: https://github.com/elastic/elasticsearch/issues/118151 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT method: testEsqlRcs2UnavailableRemoteScenarios issue: https://github.com/elastic/elasticsearch/issues/117419 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 2d4c105cfff20..b01e12fa4f470 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -283,6 +283,7 @@ FROM sample_data | LOOKUP JOIN message_types_lookup ON message | KEEP type, client_ip, event_duration, message ; +ignoreOrder:true type:keyword | client_ip:ip | event_duration:long | message:keyword Success | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 From 931f675891a78cdf907e411cbfb95033a55f7c36 Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:25:10 +0100 Subject: [PATCH 124/204] Update connectors overview diagram (#118261) --- .../docs/images/connectors-overview.png | Bin 315897 -> 0 bytes .../docs/images/connectors-overview.svg | 70 ++++++++++++++++++ docs/reference/connector/docs/index.asciidoc | 2 +- 3 files changed, 71 insertions(+), 1 deletion(-) delete mode 100644 docs/reference/connector/docs/images/connectors-overview.png create mode 100644 docs/reference/connector/docs/images/connectors-overview.svg diff --git a/docs/reference/connector/docs/images/connectors-overview.png b/docs/reference/connector/docs/images/connectors-overview.png deleted file mode 100644 index 4d0edfeb6adaeb9e3dbaa4d18432a3dd952f531f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 315897 zcmeEuby!qgyFMZ+2FlQagdiXwB`K{)cZY<6h;$8Iq9`3AB`q<8gmi~e3X(&2NO$MJ zZ|(8(ocDaMp7UP6zdx@F8E5uhv-Vm~+|T{oOFu<<37ku$moP9eaHJ$3Dq~<=yoG^* zwT*oqyyGU2a1#ULiinA*sG^jpD3zkE)iV=wBMb~lzX**BnkwIj5*~$xieTf4h)B=N z;bxvm_3p%!T;qO#M-@(T_I_jV9mX;u)jN+xHN*(`+kywQ2we5=WZYgd_?%lMbw+nD zVy$hjE&lYxZdTa1J-&SJDn`+?PChSjF9wx@DwQA3#mBeOQ}0SDoPk}rz$tQGT}Q7b znvk6Q%;W3kr+Q<$7ksm;r>KyJ#;2alF>d#Tu3_9Foq~G^v0j@xi!sqFke`ZyksQdA zs7w|4MANGT>pb=b=Jj^f3DfKH8WV!HvHP0b4Hg&~88+o%vKT$A_Y;)9wtAB;y5sWu z?LUBFaGr0dadv*UkDGqq{H#R-ZtM2@1>yOpCvqI_lJ8pEZhY}C{~92j#HJtD8o%p_ znD&gS(pgBl#-!O>aO*>U!M*yb(eLjGSn5r2_x!tQQmFb~?!O9UE%hNudnzT0Pq3SD zlQj{??mYaa!<60G`kUO|yqe);*FXF~a(I8f;X8The)^@$yc@Vn_p3xgDGAbx25-M7 z__+O=`>Eu>9a_&wrng5C2NkSsQjZyS^o=j_O1sNe-)0iJXmF{4==GNdN#EX^4&OWc zE^7=0va}dTTpf~3%`ezzrV1wXeSk@DsQKj0EnamK{m3X`UVU@J$WZ*uhNDXbQ=)_i zpNE`uoR6(p257x}BiJ%%zA6Nh#9h|MB}t9u-<743_ElMQ!zIRQU~HNqu(2%arr^Q< z@#d=&hp+C|<>_$4jRwpzD#AmjH)T|OdN6vu@HU(P^@dlh=U^{zFwdNod?I(@od`Fs z!#4Ns7k)7Xwh7FOU$HPJlAfNIxb@BT;X3IB|GGrniy|kTWf;zR7z7VamT*6o@x8d> zZ1utI;Z-M08w%_T?BkY0KpS7mUxEf)jJA%PGk=uFg^4_b2QL0w;Cg#>l)ATE~Z4*A} zABt%B-zsgg6$lT`&b?i9ShN&nEYLZPrWDIw5IuNyZRdD75hc2N@CoO15bx28mFD)x z^q=eoo@B$=J{fA__XU%`o&3Hu<)Zf1|KK?X8e7|nO*D08Wq+Pz91>R~w z&z|+Ev3t^tF|c%|)N{+R{mn&T@+^!vi&-h!*O#tF>0+-l-+#p~M@e$l?(x~V=_?~% zJnI+kQ(bxJ~hAfnGUH4JDHMEV;&#xo~Uf1_r=r775l znYcxmeR-)-XzF2HFo*SpvX`P8SaCiIO~#V}>&5#HRcEZ~^CqsX3mubrNweJ=z^}r- z@!;;wcWkc}r891s)84oh_tG^pc0E|~)*@bxAB8n50=Li)=@;O;=)l4(cppS zm#R=ee-N51siS-*GrZ_c(K~n`{C$tyVN!CILGn~KUUpa(wK9Wxrt07aTh*C@7L|yO zr#J<)g8?_hX5Xd1jS4Ldee{+1+vtxD0y%%KMjt$dZ;0<{*@zw$d8Jk5v5c&&m`sjX!E+@?_|%drHwD@0i>f!qSH2gq0*ZyzTpVzwcA`V>4^6XC|s9 zoMt1P^Zlu2!QFw~-;p}0<)T}0vmDG}krC>lMEV~enicj*$GHxsGFq zgMZ}_Z??z4%#*d9H%kXg7q?rs9hP*FN2wj1*Sb5p^bzA3o*fuGpE!KkqYXZdO@!ZH zvtF4Tt9Pz%eX~a9F^tPq#1Y69XIVD*Y4p?h*MZv&lIBHC^QlUC`MLSEHYFRErc;^( z>EQVK_#eJqdS^)2ah>l95lvKnOjpx92Nx#}S452E-4!;J`7?{i$Li)K!^;Wx8skh7 zp45D)>aFVd(qItScqM=@U^w!r48kaKhOw0+axEu2CoFsR5tp`LVPD~_Cb3q()_C!U zq5VPGlFa;aQ<);8q1l1l!s@3!L*;`;gKC4iMG`BcE3bE#miZ-c#hrq2gEFq$)ZcIO zno*oroLjF`>SzCwKak(GF(Vo9Hj^eQNPuHA_Fb$>E|MsnKcz_gbr2&(@7=}X!g!93 z@|J_F%e}M%MkO-uTA4IlHN2Kw`eQBOrRyivUS)x{F%Sm)n~7;t1snTN@<|S55w8|oxvZ-nbvi= ztst(g2T zi(cZr>^nbtjr(%@mE6m#mu)V;r?3?W$0enxxU+q}*|&bW$M*1xz?WxlieG@jU52)7_8hjFFVN5v3WsFT}u>Nu9~${B-mCkIM|_8NP%gRFzE) z7W;Zm^6wjJ1g}ucap%HEb~gH^(}RV1K0ki2>P>@hf?tlmmk#R=>vl+WpA@$-D?=^B zZ7*&MQN)(|@4Z~>(Eq#!+s9_aQs8jA%Xl}z=`^t=c)Q(cHiJ^q0BSKj*IUL-C2w*+egL&;{BQY$J=FM`jrh-@23CgV!c`06Ok1q zEp7FEFQN9ynHrI@B*PD_U$`#UMr^dUNAX8!M0#p9Jc6I}E7j(n*gv*unhK^}mFZ)p zWg%p)uK8{?xO=J1KiOZRwfT;f=4VZ%lBtyzKc)_~3Uy0OF)do%-eNujdn9Q9mrTE# zR&4REwwg%qs%ib=>)y5gjecFdwb~kwS6y#5aT2*N@r;cS6l8vA%0k$3*w{_Z(m%p0 zjXMZBc(kJaDre?VttGz2(hx;ubxvPOiG$B%Fs=Rb_@Z3nxw7~uhjXi@S&kXwMUNaE z4=6rKRLmWW6;=1|yE|JZYyl7t_Jf!J}tg+u6C==dtcsI)2Te%#l-PyKDgdZA)2ZZoWMw3dNFG zhUYZ1t=8Uqxt}`EUQeOSW2>o*Oqb_?Cvmi;dS}sLa;3v@c7%3>xiZ}*Z?k3+6(ekZ zIxspjX8+^<#H(E&n`>(VQ-UrVyr&yct6}s#^brbj34KC#9u8&nWpmwI`5l$|zWQ#f z%qW$uj>RZ$)@#Qzry8#Mw)%Ta-#&Wv2aL_v4(|5tSeSXjPf?D}YeKcGM@gqojwEKA zITFK!1W^Jrmls+W^Gfs|)x7qs!EfkKDENWy`T6^7*c*%s z;9q#)$0-T(*VPwqC7u2C8fzPThH+m-R7wi`Rxz|SGP1OLZUvVS-M0sCT(p+dvcte2 zqJ@6WNGac12jdTzsA|GB&%+5UR#Uq1TxTh;80 zY(=dsz?Z^>{(Hmzb@N|8{MU^FtkARnWi5U#H2Phz(n6O6SpRF+gf3lJorjF-hRH(( zRqz{_8T1F!1^jjU=Wp;D^R=DpS_H5d5e%t^_f?(FERSA@rckREUdMM~{qUOlL#ZO> z!-&$U ze8%h6_&>W4Qw<+iGK)0rGVC8L94-}A=nt}gbfHL+DJG_rS_=H;|GpQ$_6sY_?d(5W zGgUbes!Y>xmWTi6K0{-_5DByTM;BsvWnIL+r54ov`k!qGuH+`pugU-KcPMh5@k-57 zs=)m}%<^BOVM^8fvkS4Z=%_+fxul7y|B-D1Bj))>7yieH|I1+hW5mCf;y*_G%Xj`0 ziGL-4|3uh z&{JJguXfp4TyLzHB3&srb{#q|E-|bS$D6N(oXs88aknEjRX8$1yhoI`a%zoluqN9M zGP-GWmG24|zq`h9E`q$;SZ2-hQUwyBf|yTWc1sVk8agJ9@8%hILzX!g9<}WhrX3y& zk(QcNC0Y%sx|ioxkGvJ^a4_ zj8`rmuH+)4^BygT8D15*P8*HHNM9IDw_%82WP;O_AE)h)ubk6?97MIL!TRbQU5wZ&v1%2>;a1%K|YnoK*}N=*q#sziPZosWL5KXwna z58~WVI{IT)j5j(*eEJefY=U-+tjvaIzu>j)lvkskXSzQ+8tCSVSu%Fb739ier**f= z6<*1->e!mIN)qf}dp;aIR#?3Ch(F?J1aL=Zio4ME5HU{|Z9?+5)CwvWP_Onvi0 zc_s>_AN55^%-tJ7S8VdUKSOT%aG=Bf9$SQ{>Wm>wm3)Bs@$orXcq z{u~);-<8o*Q;yOZkF+P5RUQkj*z_0+w%6Nq4jP`_)3VeM5)PIy$<-~gaLd+-3W?}< z<{#(St>>?;tRlY|cRN2(^vxvYhd9kShy36YCE_JbTU|?JnMcKQ;e{%(kyz!5~*(!%r$xl30pN+Y&e{(UJ7MAot_R6mhTYU({VXm zNJk2v9&~JXie5{*h)$25ZCZhL;Xw~FKpUT zn`ZKb$VZk^DH0*_te+<^ogNAl+!r6-UdZ~GD{WQ1qYisUu5-Zu6t&ZD zg)AriLviM%O$ds&UY-^}7%auV{7JNJ@6m;eV5UjC zSZ}UTCbSEm9&gXaS|QXlbc^@-jmDhka;k$DVSgw)p1%USMHb3=6YY9I=h}simf>Jx z>A*#^qQh3UkqQdcOe)0;=Lk$ zA_nQwht)rALgYEDFzsC%^x)XCbk^@123rP;=JWh|s)5)x=*l|cXZ?EJxQxp#?3DW#+)Sm&377UH|Mh-or&bYcV zo9XT*GAo@x|Lu)t%5)M2sh_(Lf780@x^XJxe+wYMvLG(41iDsS_`?p)`+^;m*#5Z- z_B7VrQuJt#y+G;4hC|NB*~V6X#eQp<;^!{tO}sdJRWh%7SGxkZbY%^P%OB-9Ugs|dk!4v z+x|Xwryf{?U4Ih==vI4r>lTJ|f(`+Pzzg9#9AJ7qUNf|$DWg{;&4J}&tZIBlB*C(N*&PZ7x_iL)@@ea~t z=AZ8K{RkE1+o`b>|AX-k(NKlY$ zS9UNPi?s4Q8itR%@AazI!r%NMFopwzZ6Pgz27Dnx<5ho^|IVn#(Fz(J08i3PNpbl3 z+1zEfHP>+&<3x7{ZNnfgzSG%N$v@2!kTyN%Z^@yp(c|3tD5uNlrIdJ2)lJMWu(4ZUeHY*A_f?_ohN z`8=uDWW5hDvSie{9!D2>#u36ozc%BqFLJQ4Zxv(yV1qvE$*Th)DI!>KcYxq?>UHC| ze2<#~_t2k&*X?%*$uQ*>DZ4CYrgvu-wri`nBd`C#gmA%T*PKOv*cn&{%p-oxWhn-!mYH)c#~PIe--E9U|+ZMBChTFLK8bc8i*WS(EjN&s<1) z@}KT_^V6JyM_9>_%hGl|~ zdMYNWNWF%>IqCjHX)Vg)5Cqn4Kl+3LMdMe0-GaY-L61L(6ggEF(K7*->jW}6I@E%^ zp{IiC{<+||m;hPnk>0a?>i;=A&Y7rwN>El#(DPgU?y zT^G^&qexUXpB&IF!&)8QF0l38w%7vw_TL{^5%OTJ4N^FowKbKSh%w8Rx+b`-$al0{ zXqIDkD2g^P|4#G3P&kMgoxfW&=n^h^K%3d+J@i3)1Q8bG5lzA5v3gFEueIlch0v?* z^$iR1lI=!f{Zz=y3S9XAvP^&7kR&2#A10aS&!P=AuC9D8A(l=yIuz|mOR-nc$=hoj zwd%M~y*DOf9A|U>%AcOm4tmBtX*inZHN8Laoufk*R6WvQZ=IOo5|rp=u6v4^p5S82 z&et8Gp?6QK`VVP}O7Sunt$3Yi+Y8!lQ~8M>?-b-Y+Bn~dD7RH#Cx=3z7cMUZ7u@$o zqrgs6Q2pr{|J~LACW@L(a9xEDJn`v$;P~?!pcK2^C4tU$f?MA?-t`XubdSG}S`31c z>Se3@=m!Fp94xT;wP@nyWrXN8!J^s2K72k9n-f!?)W1nPK^>*PCeqHJ7YH+k!eaWR zhH;gCT}NGZi{U_y_JRK`lR%`o*@Iww=sw*`$H#NR3IVJ&PNa9AO6m%o_IS)&SDzd% zA;Bu^96xY>O{g9{R0?+oo3nk}Yu}FZWxS}*RXLEg^g~>@ul=;IzZv?dKCttnT2TeG z$*SCce2UsrfQ9)z4pVmb8g&?E#-S9dDq9D6KX!N2aeQ&Sfe=K`&R6HI3hhrARFufD zl;2{`Z@?4Wml7oXec*sgfVi=`-?#l9+DJr29a#DI>iX_G@UJ(0WOrP9r*Y2XkcO|; zo}Nr~8EOmADD_TjOuWB{JMUXJ@%a_<>suP+n#W#cPo92luue<)-}eLkPSPT<7d3kO z`;c|cU&O3jtuWIDs1+*=(cL<-MK3J1H{xn#vqM78ATFhh_MR0?;Cu>)6R)my1ALOG zu4~_0l9)mJ`^UhY#{$nGFXC_+{RJMF{HtY#L!jVOk@ZVa)t_hh0Dvu{dx;X1z|Y}3g@R#%5zzN=fe4WH<}Ly&oR1@v%mq(L)P9a zr*w?3(RRY?f;@Vy901_m2FQ{qqy{O^pH|q;_mSed0CX`zlS(}1$ozPp0I8Q-y*s4h zMotgD|96?oEAu8*C?}Vy?mK8tDyr*11s~(F{)#g7>{Z^RVPP3&p}MR5%b^sW#~aCJ zfWnZbPjs_oc2G$K3#V9!S-Do@8pHPF!C@n-$J;&~ySA8>^x)s)d=Yn0O!UsV?)5<1 z)ahRbID$k_9F=l9txu?H^9j&mKC&z~aseLfud)X)G5kAwv+Oq`)6KXjeUE$OU@L9)$5H$K?U^Y3n1USFDwgV7zk67bQ78+JGVRr& zCqU#yVqvgWy(bT9YtKh9{T8uSH3xLnTkZS{E2ZP|`1JC!0W1~IV?0j|9_{uQpGtCv7Nlc$quX73ynnFpFo57Z%{7pYfjIEVnR3|xxAmI;o!ZM8LeJQ|w<(|doF zZ$61Gwh3F6y+AoUy3c%X)X}Oy*D+gQr$+_piR$;{f}?`@S0!8T6w20n$dG~ZlDXY| zAuX!A&g(1^m|dr`T{E_-ToTQ=MK9mvgB831fXa~)l?{aYSc@nEFc3;=84iJD(sY1zX93I_U6}hhT7P5c!|rWIVv9 zJ>MoQC7@>;6{Y&ZOP}CmF!b^6j;3Hlull1LIT|WCkONJ=(MHJv&;gGz488}!$(Wks z%~pbxGu5cw;d5FtUa2HrjN)hgZ2?S1FqaG@53F81^y+OTD1j;?!%$oFkKJCD~8qKQEBgB z@jw!#z}tjKIx)71HlFdgaxj?AjtCvjXvo z8}MXOakvMl8n+#0@Kh>BZCjbe1}2vBYUL}J3uLry8VOtF%<-R>*7clN4v&6d;r$>e zJg3`P?J#;5Q%XVT;OSw1?MZL$7U!Q>)Oe87dak#u-bZ_n^aN^eTmb}kX9zHjo(luK zY|XcXgO`_CB%y?(Ha;HBZrBYdt0C*Q3lvUN%oX#=I7jj2Fd9iwvJjxo|qIpIWFPv3rPcLDC{56iGM8sWg?j8~)HhauG)u^;Y@ z*eM(WT28xqht>0-6%OzUcTvFG6g}seh&QN6Zi8ECnfJQYy}3@d(}mMC=Sl<%BK+1DCX_50$XuC9dYVcyW59=PdrQ;-Z4QNTl3vFmn1H6j$H>7E$G z6sjl)a-rrWgiU!*Api9`dx3ko7I+N<<~inwXz3Myrd&xHiC|g4+OmA%u6Es%ubxZb z-}xwOMD*kXR2`?u5HuU`)nMxs+rv|M88E5$#ywA;e#iti&QES#cSa0@DZ!?Vt6z55 zyiksPN2drCa=IV#rsjCg^TZftZI+QS)kJx+?gIx@upH5I6So8GelgD$m3A&ml^m4m z4*dhKjZx04OPs65vOC1WAK4$st4zj|Y%F=6%z66bssuVkzOrJsm>6?i)AcUf)!4AaWaYT#Ol_8LoqGelnj zs!keqVk)BrMSHMU@17+%&t@IRmChvS4${>ekz8kb8FA_Nc?CBEQm1IU()=u+by;vf zM3G!)5RV!J>8qkjIDXIzVhL9&mU3}Q1KC8dJfTvVPChOg)F9;soWZl0k~s8{+%6ql z8ZJ}c=*dhe+_$@^AL0>$tZm`4)2l%i%r)k#B!BhZ{sdfs_P9sc@2yg>@X3B;3_&w> zwlzQpJcE|G(!z#MW1`iW#asMf2?i~NTF>66eS^I*hymE$g=uGP+_Qj9by`sa>kT!p zVjJEXwYWva;o`?(_#kMiaj$?)voPhS4a(|GjP_p&lVM?aK)caaTzsc${mVi@D+?2% zi7BzLOPq|$O&_XQ_*ToN{p*DR71@o$GoomzZQfj23&?n~F-M6GE4NqJAheKvC@u+y zni|-&T$aPIwalGS^`p_oojzU1(JABXjjhrm$`6GgUKb4}2JN22mBRCKg@`{r(ZiJ;6c zEi^F=*sQ0Zb|uTbPQ!%yK`+d(OS29_WC!OBCaWrdyconY{9zG8$_9K{*koK>%JM?B z z)MNJ}PxhOI<+;fxKyjiUm_-=G1N;UKI-kTebdCHbx!y-F&YbnHCl{=B3yHQhuSM<3 zfK;wgUO`gK@Y0&&JDVsh%%7pm)~`w=Zq|gwGmq7B!{C2+sxOjS?Ph2q;lMO#eF)N5A8x z4Qbwrd6I>p0Mt2yxe;(cB1h+Oy{46=67KBDS9liI9PB7n7IqvR@9N|`WfLzKCVeQ|eMtLK#Dk=TPc}lxzTC2* zLfi;|85$78iV}t&mAIkIG+`#Un3EyY=1Bl z74@-uul5u*<=TeC@c;x#W8%FKflW!#N)X5MqOY=PsJ8O!v*z({j>-o)5>KW?7cfU* z`!sSo0y3*b4QFY;9bncGsO+O{>`=h!*8|Y=&9rk0*uR4q-SXfyS42yq#_8yMI=Qho&1KUrTh>lmKLQqq)6-)4#HeI zwm6q6_t5^@>B$ZFJz19w1WBTf(VP8qYP2ev^UC-Rzx zQDqFuk4tMo?t~$|cAAVUR_*48!xv)DL0<5{t2gBF=KdlKrUkoWoPGC(S>7 zKHr#@wGe+4*-w68`4P4HEZZ$_gClWVEs%gvxF4t-+D0uLP{68*CL5P>D=o85;&_I= zDtrfYd0*9-~~o z3=Dn&6l{zd?q)}TB$Q<*?E%#CUXGy>>~lJa6ur2dKdy{x3qa^<-fuSV{R;P+mLHd} zpXQWJ`Qxk@9F|~9iaD=;zCa%<&UQ#dqn(rAM2fSr)dZ?0UAKC6_pxiBG^2hR6{2RQ z+jjEsxiM%LzO!dOBws7IG4wwBb80+j&SXbVu}}tN4pUWmmWJiBTFXeu-wWJl@~H zSxIJPZ_!LB$cY)l;dx)w^>(dfXe}4C6yKq)Gqi7r=eHA>!b&Pu^HKZY_2qQO5&IYB)u zB~iq=B{SNCSzP1W%sT}kDVTB#&A0E$*JoxyRf{GZEW1AiFAN!QzHg!xSGmpyf3EE? zXhcRCV}1%M5f*U`kV67uaB>fd)5|8V8 zTszRr9(M64s?NKsjihj0hQ+S19NgRV>)mee|GpILR@&>~2&)@@7HeVK@1Kbcb#mXJ zcFkpT61R4@pO_X02}c>mdb?{;5Da2flS9D6)I^m$VHrH5scpv+^& zl!86XjKcdIy~!!XzWBWZpt&qz583w)#~aPW{Xs{gfSJMBE87a@?^a-+Ta+53cgR-?@Ns&i zY>!3i7uteA_beGZx}nD!R?&hiX!K)eeiW#QZv^H4=*ZQtG$FD;U1YCBdFrcVIHgsY zrCKN)B+9&$Q>w?P(Ad|_(d1|g8YrBcerrtPJWL^+5!c$ehs_&(AI&VBl3s|h1k6X# zSLy)x{4pR7Ys!q=c%T+|jXQgcTA)j~s3Xn=QVa8<$u|oxTf$x0NA+0#$2!y9@xcMt zw|;x+RNk?orExS@CAgQMZDIH$Cwh0YK{?vebhPzPi!9;_TBJiv2tg<%Y&&@gJV3Xn zsAZK|X%YLN`&CRz!SjHeU0T)72~y`(+C-WqG%qUoR7R%+FUETsD4a_Js@W=Ik zt411y!$dDdy2hIx8eYP=5BXfn>|XzpF@_2`YqP@&O`pEF0vF^XgAV z${M!Sfr_L4X)nw=09W?v#n&d@Eqt~fRk8c$)is~Ie$$m}a4Q>oqA`&jq~tXuc?5=D zjqz&dgUig}pe>;4HVso}iZXSH$|D>88R(~s1x}9Jj6mL2U=PzOmge$k|JWN9tTRU+ zvG(-91LfaU$7f4E_s^K`Vv*|eVGt<%hLG|%$uvk!xWP**Y^iftg*W0m(LCRk5^xx) z2giDMsOF({LV3DD+4!8J+yd*eywJf910GCb*fU>~zGP02FjbA%jkH)GAi)QSP8F<( zmKi`;*t2UxrC*OCIkFnmNX}wZ86RxB#j(U{vIS)9fTiDQDDT(zbcZoAB-Rd@eoO-$ zl^>7pXWGWv_i52rgWkKCT3|Rn&1!fn(4J`r`i9rvW~bJW#gA?kicm_X&JL3?Z{9X! z=Ti<0Q(*OGMQ4fQ(+{+_>}>Qn;7UGOEoL?~Lh#EE!oU%mGu zv}oMpD2?)X;)3AnL5yn>X!9^Z%wQXN!qg$CrW^{o|7g6;|UljnCD| z)DZ{%D}V=CRj6tOY~ZQI{0Y?F^Jb>TBDx<>^6mE6*4b+Uin6bW#%0PIdR$;Se%1;) z^tn+oI!xld{xIJdMs~r;@sr)ssZN6cW_8)F{Htls;|(()&2hPO3#fC7qUo|?)_BDk{9$jQ((_HL`0ZOBZxdt! zowlbbPkGl6P`6AF5I{IbZ59s)Zi~kFttH%JD@e<6)`)W zLe&zsVIzU9TtriUkgG7*Cb9Lj%w2Xod1uvBCj`mP^PUVPFQ8<_!v>_Dz8NfoXRwJ1`RdnDzxOQkzSSTDGu>)Qmvobz`vxpsP5SvJ1I;ix};D zmlcke#h#~^8=Ga;vvUnWn3_#(0+igL9=nB!Tyc0_DfNt#g|pIFb=26AGUkbZDtG27 z-aCd`B4XHC0uPh}4G2@*#Oj_&%Wzeb?R@|p4X$zB=}v=fjt9@7Hq$gqX_;`bXZxcn zXZjm~3|xjWZJUW+@z#$@e?vQ=%W(@(e4FCG6GvlH1Xu9dRsV1|U8n6wyFIUqCwOcplxy0nFBp`uycRkf-m zMX?bOc#b;kIFG%{4H}XoUusP=s^9^|Fk%6aEsU1VpU)Ae0xTVy1R$vx#ZOTp#X0I1cr^PBMHV{*l7Q36>n3_+{uW|AsgISfdOv(Yi)e#d69KS$ z28Y>^U&wRZdDzLvK(OiSTn62YXmfZnUJ`x9FQ-heoXu zml}$+VJ`PTG+AN{C1B+c_j3jeg+I)m`s)U0b`h(1G-lj~eaTe!gBenXvb${0Q_mCJ ze2zLLtSML;48u@AzFMD zY==dexC+^Z1t=^QofL-x0I|WHLe@ zc?L{&BNbUQ`v#aqk1kbadR6_dfCQFsdyecWw?TuiRZs)q>p-(do>l}t|HOw7nHUkm zRZRXQ7u!(>_yogCHQ3@1s8sx|rN?XBWBJnR?O`MSu-6Ho-EQCv+Lhga+#g`W8ElH^ z@}AN%{E^UpySOj0-DwuF3j0t-V$b}=LoVVRvCgOHUAC7lR<$UH!tp0nwE$ujWy%oP z_pYz{uayj5$eQxX+_BSZWGOjPX3v>FPi7tp2`3y9m$3eTHrR>*Hb<0vT0&QvrTD%u ziSt))LapCv7t*O|wa#DU~ z%{vuYf#m$$jhvd~*ALQT-1CZ0RJR6WD%ArA2vc$+(izQjEmU^cle;NT_nQsiIf&p= zRi3mlUDRmFwycVax#>xqf=MD`yDEBBBI{{1ngKhXwA(M zSMMBMbh3ruRZ$&OfO9^sd%+1hcrmN#?f+170~#P`y?IeHg=pesA18 zDPSnU8v72kt|aHu4W|h&695`y^RMzg`}phs0#N^nd2=qtfrC&Gl$3$sagM#5=kKlK zg$SPHChR9~Plsi|7)Rnf1+3F#DueOo78GNq$Jt6lE!Jbf-r~H^qrazTlCA^Wc=lNJ zr@dZoV08iY_bgh!$m9L<07eV}FQ8kI;5L8&|`^^U(x~Sd;yf03Xsy2T&P+C zZS>m2yp!Je(D}4?@_Y-a+G|-Sf3Q_@NFO#Sv-cIf_Z4-78ai)JK>iULZF@0?lM}JBVKLHHE{!9}mN;4|t?DPdA!TTvVj`A_+vR;*d1&FOx$niZCzV z2?c>mxW`3fC*<#x81&AfB8Yic^(%fpRNpxYm-!DxpwW^FI4CYT{*|JfMBum3LeD?+ zf*?~4(9kacjlq0FWd-5yO%3;XELgzdxFWQt^Wi%|- zY7>=#60#fgL*C5-00?Q9du%m!g8s0as$eD$Z?w0A7F5O1!&lfcxkCfS% z%tUC&o8^{2gLDiK&R^23Pl3Gj@8gT)K!&~;+r^6B+KXPG>reC{6;KT+bcZSU&_uym zD-gcS4ilO3WEBLrWJv)(y@*U#aDV0G0^RiYaWkZVGsn0w^Px`;7E_~73BhQ@Q^C__ zt5>8_qK9q9l(=E@aIy5urG+VB6a#chD414&@Au}}1L^&kYsuZA0Pe*??6gwc-wo+Q z0ri~UQ|Y+IvKDd`?fa-pvT?lZP*Z~8ffP9NWDYfamWb=${b2#*0L9$9z0`FNy#VLD zKEJw<4*FY6D?oH!+)3^ZJ)RdIA=K23fhsyqaHw`_Ej3zQ^^e!W@<>^IWdZ`eDDY*HSX6M3$-yh=aFgVa*AFGj;0uWe2o)(T zR?@=glqYiz*i#9-?#u-Om6_~MUoPnm5%ZuAvK-cmC(z$Ka80 znW{q313<-`Zou|w{W`WX{~l98a7Pt=@)at0(X<-aw+dsSjP#$!l>Xgby(nlwS1m;^ z-3QuDuS{m^R_4;w>0o{qh*DVa#+L{BbvsDEj+>gch)%z1hCwm2cRLfMCi(Avl%;9nYnY zhbwouh$#F)B`>HA!j?p>-Ziu^Q$=P1$hhQc4}0^vfh?dKwDIC1^;{M*?V0+~F%g7x zC~QD=J1B?qpl*07(9-9$V+s5`U-UOWhC2c`UOhR#gXTIbWG5hB9bcBe8<(SKLZSJ&Zj_{PWy8EJTLn52{{ES$-MMJ@bxGI)lu0>CqGpCS998ya` zf;~7OdkAQ#nG3#BH)cper>L2X`TivRHPrx5=rA6X03ibr9z5U2U!lHiZtck-n(_rF zvhU`AfMWvs&N84`{*cEdEVA+tl4g3nW34^D>kcw8!)~m$gl@Tzm5++h+)_)qzMN@P zNHgEi57a=ya$NVu2)90KV${-Bg%0KC=nPiMBf4GsQI2ueJhIY`BhW)_wa*`QsvY5m zPmj$wIP(w9Sj!XztW=GwYSK{#U@PQFKc|zn)VSLGqardmDn!h4D$Z7SjD?Zu^s7e~ zcg^d0*KjA|nYnK}gH%I5rVoVjgTvOYga4W_1htT!STpH#BcM*dDCCo3xkZgb+`Bxo zJ(m7C!l8%(9l`uwG>YqWc;?S2*WW@9XS_b3B)^M9Z;=TIXx zRqY>|3+@X!4)Hp63)&?y_LW;quZ;9JHQQC%x{uI|Nz_-fE!0(1AEGh5f6Mc-tGge0KMVSbPU{kI2A3K>iP z9tHjt9VbpfG3Ry>{b^DQCZu&M9rthrbxLJ7I7JfP!9}(5U6F}z#k^;v= z7=sgJ8DbiPI{+<-MH_zcz9?2llSi`*8-~67I-DOoA(7oWTrMX>bfhm9i+!RRIl#I| zw;~Zuov1GqS-Qp79#oRAf0gc`B-MrMEtB5Q-z~KcchU9FeY)Q{XV#RT(IMl_$Vg8x z_oG5n!o*TI_iY~jO3M*A?XEOMI#Z#OxRfksHaugO?9L+BJ6F+UvFVXbR_a%QT~U0YWMq+S)`Yn@!SJxPjCE~hOPS2KHzZXZx0b__J|`}a$9R! znwKb7a28%{8SaW2VC|pbwz*^8(*LcwWrc}8J&*qSL-+7au2ze!gTdRf(^>^~<`RCW z=NuOM3Rz6MG8T5RZaJC~e)f8{u~jVnQA-AuRf(RKTU<2j#GmVr=Oj-GGdB3DTI|PF zPeu+3l%s0%-R}6jGa&D{v2-KU-#(YA)T}Cgc*ed)JdVuXxweXhvUOakh&fW462TO| z2W0XHp$L;c@em5#JK%K1Gg6R7{A{S`#~(|E&QF2!sD!?X635;5fKEbRpcd#?79dg+IzsT0U_<7Jmq=x+fwk5+ zut8Inhvc!BC~i}{m?f9D7| z`uFbHd!tB$_n-s_Sl7Xte->kcFkrwGim)<&(_%kW*)nc+HMvNG;R}!0Z6jr0mY3r9 zZtMV$CYsDBd*+=nWqiADn(0hteuQywlIpRnnBV&UqwCA#p=`hZnM_TzAWO0|S+cJQ ziJHp3W#9LjBKw}Lj67wl>_W<3%Dx**R1ykNvM&|cl|7W-xrcgszQ6DHAD_?jGMW3n zu5+F9KJW8B=Th`5x1#xq41Z{#?fd@oRq-n&ahii6ctZ2St8sBE!4l2H{P_)#h6+76 zy~jGl?t@BGYR35a6BXj3fm)e$x}QaKY3%t*c^_KrlO3J$yH-1NaV63H;n?K-iFc0< zdgoX@b2nG@7@ZpXC7x$@bu*4A#=YALqA%*LNz-;t9lQJAkF@v)BhVnMO^WDkgQGGI=x@m-eXy9 zEJ|Kjg(JS(H&y<9S;g+_KOEb(hd%c4{iSPDm1?sDc=Fby8A$0CSiu< zU?zlBnh%fFJg{7hsuac6C%X#eUi0qZ5I>k_bqJqVIi>*{nPShIQL$udK_S*Z zp!2E&64OLP^Cb@IoJY`ODT7GI8(s^DqM#Nlt?)yuSD1Q^^Vu!v_R6?=Sae1o9BH7k zXsAw5LfXrdA?*f1)vja$%b!_aYO8VxmRh`E`airR*!waxFO{~b+99wAS;{K(QQa&D zA4qXJq11LCDKuCcmmVZ=XYE_h5&O`s7;D>Lt(%Ui6sTG3kble-C@5UBsJ&(|dOx1; z>r2cjC@)_u1f3WyZye@GgLIazj`VqK)?DbYp#Y6WDD9M?b#t|0N zS?d$xUFA+g>@p0U6AkNO6V=uUP)iD%sF9wt?GO1sfF$HH{s3tvlZfnF9$&K&B^szf z5yF0TG5`8ce%T*iqDiCyP^YTRM`8>cX-8u2h0efgstH$rJL_T|6vJL!1O*kwkqiuv zAQ1ENebSd^Jnyn~zMwynsZ5@!75!XQi!t4zVbSLTzg0;BABTlJ#uns60tz*u!#xNQ zG6s;9uJgIPl0#hpxqJoht%UHh_cNzwsw>76O26O^vfUBot&q zwLF%=nE={iWDDnts2%;Ql3`3pL zfJ(G||2lHww#sAPtnskOcKWDSK%ApHzrxhYyDw?&i+bpwwaGf>db z#<=TRE}oprVC3U)X%G((ZPt(Jz6J@vJ?+gdsY)Dg!TXszz3 zOZ0FGf}&eqllQrbD5?W0OXe(L?TXN!qyv2(lmEPR>k6)eMyyUK#_Oo)Ui-xkpTe_iDq{7c}K9%|JRYZd1Z_CD=M-oq7rw+GmzYASM zG!a18wj`euOxk43hFY)|?q{CHe#$AoZ57`wZPds-OnuJIexXmrC9t zyuEa&(xW1d{S{#8GTPcfv!D9z4x0*yzBLI)G3XF;f$q130vT@{RI>8B*yKr^%)d|b z=ds8Lj=K6?B%E zhr80Hb0ICy@_i&#EPDe+iIOnO_YK;U=yj-HEe3jr(9~KpK_ld$Q+`*CU7NVVoRJnK z1BzJURPp5hCB((Hdw>-8U%nH^y_?1DjpzhO26+_z=|NIo1!f1`pr}vF2s`5gAz3Fp z2$}Q5KnOo5IOaRFZcg(mSSx}KC97u%c(g&>QS|5VY5tZAv3Nvy{oIQ~A)F`6|KzXL z&tdI@K)(&?Fv&3f-ll&SX4{tV;5qqj5LveW^CSukQx9nv2rGC8s_}=jFh0f|1h$zN zBrG$?D_)Sh2Xd63n+icjUIQf>T)_xL{@8kMd^hl>!Px`W83jlzGGwpwLOc&X-mDC> zfZ(rea52gK{N?{UR}bG$jD|z$cgtSD77PTN9ixiPa_R6-9!xn_Q zy!+*%k$&Ep?(qaF@;3fK|nrEfyf?W;(^0Di5zdxh$0PV$A#t& zQDDSruYzHBf@UWI>2J3-@|no&gd(CvP?QAp&6OB&-4|aFC>ED)`TvN4;XYKnxsUCk zW0%y{5YT}%?wx{4mpIIlq2oxsd-!y%At*h==r%jwLo;3iv>rZPL^?VdaLweI7_Np* zk>k*}mbLou05Y`9+82T$GCx5lbV(p~DF<59F2P~7PN*r8ZDeuFsJ=4;JM+{k0dxno z1GdMv;YKl>jJF+9+fL;@u@1b3iM!}oC1hArVNHh$e2aXN*XW9k-uGY9b>V5?Lr$8p^yg zw#$i>$PIs01+%wDpOf`wZA>>Cyz8-VM{GFXK4hbYAj%9_Xs>jbNtOoz%AuCZ7k)HN`NT_WIbAIB)_ zH9H+UzZrLTbnVsDs#7!~yevf8X&TmGi|h;=dDLk%ktX&|Xy!4SRl*_fj1r)O8G?sJp^fL5_<)$LGMKGy#RtkeT_lg?qI* z+J`;7Ln1Wok;!b+ikgsX_A2^&hfu2rdCx~Zk`DWcX^NS0Yv(?D&cYy0nD1lrNQ$cn z@2ybRcAl90Ce+^s=X)@Am z^prI#*PP-y=Ex{y_uYFQxIx9@u}vErCx=cf63k@%I9Sgk5qE>hUcvjBC^QT`}Zqv|h|k;GfwV*DZ1!{8Ls|p4~QHWTg!r ziF5+TipbhCxZ(`i9rSh!$@RdnCuw&x?gL1u5$*p0>Yq4AXd+_Ipxd-^LULeIo1Uxo z$E7#P|AicNF}vkbI0Lx~U(#6?zVuYv!O&MIBrGa#_~59(a|LghZYLdhv{*MTfqx_t z^H8pWb8=pKm)57Dr|JHtr@LsxLZ=!$VGJbCkNd#-#2HGKe)*VRUKuh!NnV+GE4xY2K_lvH_F!?Ce^`RdIz&bo03m;cyFVNl zVm5o2BTT{`c8$B8;lxu_xc6|=^2TJlU5-B1s6>n9ab3{x2H zhTwS`{YkUkbPb+CvHC{%SnHF|Jj1Na4V@;;Eq2o_h_g?@P?2q*LYO8fV-o}C3fRV6_k-KM&4AJcVv&p&G)K5bj7Z1qn5sCtIZC#Uj(Z~o!$vcKN05E=ba zEmPblIx3ZWtg0K|eSNnhx|%F=JfHWma{VE#XyMOl#zStgjq^u^ZJK=9WFwTxl|mAD zc_gk0FcVRfr=;D-Ba-&5kL~jBOgwI9YI*WeuF8T5GPJQSr&K%}>)**DX8K)kG5)6N`58Vn1&vpHc9cfj-{b#$8lO(x;V_$>k|!<_FJz^}YW*K0Ej( zj%W9MSzight4&na^J8n*J-Q`-t_^J-mt8@{IBMSHv?z#biO5LF+$`eoW{QqUwS&Tx zkcjQvV9l7i@A9;$x=&smFPdj`5zRgNI-!#;BZK$RT;b85*E5eGZyi)dMy{OdzcDV0 ztN|9Q=CmRT7XQ(cJ)T{Id|bzZEg2YT&Q#%b&W>FEJ}IJ_B(cbhF(Oz zGmVX}NDoMB9uSZbB#93ry(|XnOhbUeXoZ?mXO_C#@h1A9jBM+MP)4J~c=$NM4Go>z z=aWRN0^dH6?UQiqw*m1|@ujzkMx)KKZDtXdxBtqP`H-Q`aHWKwYrdIQw*#;C?(=wp zPRtpTgqzjaf8g}iBxwQbW7oM&My_gNR5JTeZDlBLA^4RCKj+zIRV#3F{#g zX7xO7b1r=s!{jC$q|ttH0|l|U&6;N7M4{MG+u2y}PFG6J-b%MeNuc0o?96DRft~%X zUg;O@=;P70@TE|uRylLjhV*CzMl6>1TVnv*mW5N7p}2powPgJ$O60*PUzg#F^(f9# z&n*;w5i+Zmj@W|>jbZVn+Xw+E;W4SpulqRSxYJ-ig#1$)5)J5RkH2@BjBu6mu}oro z&&w9qPjBgiVr0BsTjB)!PDG>jX(UV2)8Ee^;}TX<9?%krVtky^IQS*@Mh7xEBjj?M z5i2iR(%WP#yiT-}_ME;)rGcE-%86pRGpcY*7*k68*FF0tsC0X4 zFWj2>rcSA;Z5cH-7$aNmS={;Gsnwv(EQ?SbqL((lIG6BP3J-MF(PQTWf5v?u$A2TK+1Rc<{j1-#p zW!yZzA{NJ~p7XYNIT^v?)~sw;^ydjbPFFtt*2qv!;QvdsU>_mIxw;f89{}s3SYLIGcMzou81SW26ut;$XOlB zj{ZuwnY^mmA{c8?rxGtT^K4%M_kUgo`x32dZQEDopcO4N%#TrC*ux}NVb6ga7ks4d zA?2@u)hV3$86Dt^Sm_U@ES#aFZ0}Ff)wq2V`~+tPZgE&q7^D4AIdLD0)bOcWuIs15 zTz`li^?qo@t#RM^p~KpnKBC9d^?scRC1Is(%lg19ESEZEVg||2yi|vSS5aBWu;pej z@Yp^}SK?FUMxz&R8HdvGnoe-y<o6oB1fBo0^ zQs~#hrfnYUc(2EZ(Xpl`7l$=IWUbm%m(p7`Eo8`z?wwMK=XUzGCQxzSKeb zP9b3&u4B~yD&w}B6JmE>o;FI7@h)mnNE0ttyvugnY}G)4wziTb#&MPm<_LIkV&0$_-Zb-@an&e9}t!trq+Qa-hs}#yFn$;=y)b=ZP=K$KAJR9cOr0_m`hU{eHCDAnf0o=+5>+P`Gl6Ser` zk$|#~A3%f3;kR#V+djOl#c|YzP_pJHwn}lORJWc#x!G`=jCo3IWb2U=~sm2AysV3DD_+qJ+ z`_od+4`r-)DbL`svSrivgU`Evu`kDBW)M~LuXihwXj9J=wcJC z!J18dAX&mvR&*-L10^!M*$m zZ`eR|%;-2YHk3oykV7AtIP3K;O+MGK&^nGoao z1MQcefR`WE@Nb>`{;akr9{r0-PFp0Z^#reKGO68E`;i9g(h0LCy1b0W z>uPo%^OBQR)u`SSTIQ3?ZTp9_H#uH(4V-%a7xWQ)>-`>;7#^#JFh)uI_x67r=hDgp z!($>}w6vyH7O}k}9RON`;r7&XLOR-u0h<02Pv2yo3XA1cCQmnZ93Xlu!w9urSbVj5 zWyF7CP0>KnedK7V*gCJl>2=>fH=5aU#7cG?R=(`dUFJDEI3pJFSe%(;RgC2L>&z!y zGpe4hrru&J+Q%y04aIp@p%)ie{(fj`YZUQ_{Y-DEqc(Mvkm1t_YmcTVt{ET;%Wv1Z z+9!etjE`SCLrM1pR<_HV>d(5dMO+M6o(L88gIo^y#34n99(y%7h2u_AacjP=08XH%bL@X6EU(Thb;OAh6r1&P!w>N_WTkM#;&pa%npJ0!+$Y7OIvB=h z9mXJ71jaYSr=N564EN@buZO+ZnyJK=#*Dwg5R3BUlU~NnF}#8Z@X^kys`g$WhI{$= zftJg>K;0ZM<3KI2CrcT=#ob#beVTH5pbFuF9e^5sQ0mav_47yujuTI1+h0+5*!OuJ z>>;6!(hs_dB>YyLPr7~y_L%MOF)qDA-@UKU>+kr2;U|xs{Q=3@<)`|&u4j~*wf%t} zFzhX{H5=29HP@+&JQ{mpf$fn>EME@G!9B>D)HR@&XZjfF!`C1T6Q@a= zwFAk|+Gblazz-81&);v3cx3q@Lpj>q??pAT5fn=B7G(9sCR^W+5==SmsQbhQNJ=59 zs4FIO%dwYqn%KqI(QD%c*I`F8>RyH%a2Rh-oz*C*bo;uqh<}>-&yj{zIctbxx~rdO zIuRH_7o1VYfyVM4xG5ygKcy8y%Vqj4l8qn3YgBach>SJy4rvS9QQ-4Eo*k30@3w4C zEWh^Cr?1p5emyyLh?(zec7Or8l+U>-Y4O3ea#N_Z>>l7My9NL5r6D}_L7hVEmOYFd zNqK{4D(+<8EDgN7*che}m@TITTm1Y&L^xm+>flFpz#mQpMllGR+e43Ou-;RB!3QQ6 zJ^r)oUIdsi-ipS86MFs@!0DA{$Bep+jUNjpa6Zh)q9nPaijbk z$O4q_LMG{ysS@k@?6NMcM!l-^+`|)YmsS!{mP_l3Et>O!E_bL*xBy0Fny zdYUT!3;o}Z9V||6Hm>*WT@k*eUk1X0=Wrc)yn`Mi+#2yI|7dfbeDC>PBrLbVpQ5Hh z8LsCXB{`88aE&om$gFoCo6NZRM=`$L##w4iE5xxM^P>w+mwT8|7&#~;Dbu(;X5vBC z7{^KF2c;RIPQ`{$dO;XWB5`fwDr4~bOjSlz@R^*QBA)O}!=Wg?+;329AdYtBnb|=$ zS2R0NWdfKauR@6Guag^qTgBM-mA#DwV zQ{0H=%{3`+ed}SaTCYf{$!A!QGVucryc$!iBd8-Jvr8UddxUsbLJ(Y1#(RXWXXln# zoZ(B^qXa!JWtpl;U3~g^_UCTyDSUOCxoZAgf*78%!@N@v*~zQu#e^r=^Q8a?5qw@d zJR=~BM2xL%f+Ad>5!JO0yAr$=(&Ub)>zA^s^UHNL5qgLye4_?Eb(%ZxpYkTgcZDqFdW zFi~zj@`HfcZXJpJnfvB?OP^AybnGsI5cQrCg0kQ%7UYqz`%sN{50``Z6$unMN|Eec z#h1^|vV`5>c>31@Q`@7OISa3R;4`W5DQ!9U9jcd=_B2GLr}i|>Y^iTLx8eD7c^tTx zsEnrdp8$8(k{?Am0oX4AJ2d%cel#~QA9c=q=>8s+5!XegU8oQMZ`z-ji+61;%?6*y z)o+%};x-X2{i#(e0Ml*n$}Zo7qxClbrzMin;VP)c?-asxyNYNrKyIZlxfmjZD#PrH zOMdWc(vrcCpxbp>>Pe( z1_5WrS4*R(Jtp5vT>s@a)s@RU8uWbL<`@rnx1$8DbcG%mSZaO3^-BTp{=JhQ^AD>H zqznJ`wAgPC>mXLnp(5Pf2w7j6qWd5yObI=nOnZ`5Qd_k0JM&pRO&;r|6A)FzW!rrJ zL@`yb-nS@~q{*GK=z0BY?S`L1>JLx-+{}H$0q)6$92LdB_U+hJWMv7?)S441F#5Rf z`9D;-u$+0iCh(kQH#0yfeU#x0OLQW!O})%+PYfm_!Hq%$>8RgQ;2?Bna|yZ(Y% zn9Br6+2qY+KA_*{7!~)zaU5vSHWU8eJbZMZ|C^UN?AoY`?Z9>sMcO0l~&s0f6+srcvu>|Md)5b_yhHL8k8ZRk3={-b1T! zEY%og=>v2d3~tWc6fk0ce&ftmo#1ptahW~tE+*VvN7WNAvP!!bu*v#xE;@Fz zk=(Nk8b-5Coyw0bBMyZL8w(CKG5!ZDy);7}bC&%P#N~htu_lP!o}*UoFXH0S*=h-5 zEsiuY0AxLR4~*>z*|8S?pEd z{U{$?ICFl>SyrUg!DSa#Abl-?jME6o*eT+*pjw7W8zYflyf zay(ekT5x&m6-j^MtePcM^t1L*QQ|sGbG0*7${@Kt39(TR&dfkTW@T%1`_^~iuoY@}Q(@=Udaj%0UVn8cI=PJ}Q+=c|b9Zg628R2(ir=YSV^CJ0-L=y^1;ucnHyTRxg} zWFpx~luKANQ8PO%24JWQxe~X3y<(VO0)2D5@LUlek6lSKN9XTnbw}SheAfRiLS5>-`g5Q%&50@=tVU zTglKInedpeebrY;zi(x{x4!goxkR{P5!jy+XsP)C72B)7*kwp>v6X9KKT%Yx?N;?; zFczVclZg3{= z*5eYBC{E=Z$WM^5(@c1y7Yp6To_wQ+Q0sSBkb@K{F*^uQ+iyWkbD|@&81bCRK~y)a zL@D4^u-0f2&(T)yIB*4_{XhUDpM&1P6rapRo{Q!T1X+&ku zVZty%*{$BCk(DEDDvl7%eZCiFN{?u~DZTQc*=Kp0i9m_utkf6_0-V7YSsKY#AvtP2;YOmT+)fhenLw)4Ni-(LBUa_P;UXvJYe`Zf(l1sNyxO}FZSOyy!g+Z z)x(TrTmM8#aTqf90yt_4>U<+afB;p{(TNPMD~I9dFXq3tU2m0^IYUMmMqrE7ogFpX z-=s1OmD(~`@Jiv%HkpNRWY(+J-1n`=@S}jTHI;9NLDVs~u{X;AwxV36{q_lRoFV-l z`SESHfEn>UUP7}c7EYoeEVsae|^5^il4!aohkw*0PYgzL-uPxfHL&h@$syx@| zB2I^%q6XKmtqw44pa7&donn{X9PE5s$O61o5kQrYJ#WUg!b|Y&I`HAsP`5Akoaya> zw1X{4&xir}!&nNmghTH|S)auM$nEbnyAMK3#Stk=;uc6E?PEX>kYh7qjDDZ_bJJ+~ zfjTP!NY@Kd!~SjJaiP`63bzkAJ~(ct|9X4<0ftfWh}gnDOZYC^>o|7+CeDR#?kcc4 z7ca{@c+zqQ21_jrzW!4~WAk=c3$FthW{Qo{=tV|5&wep|Zl91#3PluEb#VA{d{<`6 ztQ#nO0o&$+3buX_zlP)&+t?)FfRxYT;}-1{H8fwQcBVELzj1X1m6EnMJ(}}_E_IZX zN}@RLmEWJnrS0b7X-YycX&>U{)QIb|Rk^y^+972Y67cHr3|Q<$heY*_tp%{4pbJ9A zv?AEo9wN7Xf#sQJ)A7k$j~8DD)4XRuqbNRbd=T`5!;>sE|L!nu_%s1T-xX>U%?b!< z2^}DP)3A$PCZ#`S3)ZSK^xWZ{x+H_@=Vl>=PIL<&Bt+U3_gxsYZm>wf@q3 z0q)_+?9>Dl1*iP*nai(9FH_HfOxo=!vF*%pPCYAeaa^+&v#TvNl^cGG!%@Ka8V+BW z44}s7ymLPrOT?)Ho!_Sf%&(Pk73z)y^0IhVO=@k(V5WhncIkD{x(>}H z`OR*>{|VL-^+e-4bU^koTKHLkRn$!UfExT%RwzCH&K!&Bg9IfSI3L<~#6Ob`GKP-; z2i^#P;*9Z7Q`OWlycoVAWy`Z;2|pIT_JQin8t+uu@ahipu5DaRdM)w;?F}cL<^v;Q zY*Xxeii8kS2@H;*;8)>=KTnf|0@UMLuMDD(>|`154zz)!V}Cd7ku`Gfgqq|FS1>ae z2OVWiXh_@Idd{GL+G#{w0_n^t9g131^fpeBV>Enrt;S=bqXGX;=Lgl_5&0LbVVQlx zME^fUyJH{^>v(kDsX1J?$!p)|GNMPiO39C>r>A%Qc0Nir1V9VKxsz+H3s8?tKt%&f zG8ve6z{kRVKS2rXiZ|-+sS6~B`mL%Cj`9^qUyA^6^up~Pmx0V!_C4Hv1hH$VyiE_* z^g$!$5oTGRk}2mqEGN#UfBviFhW$cO<}~C8xv9GjWNtY&@6u*BS1^;ND=)^bL5>V3SRgaV=OJOEg1rDJU zZuLft3;%XB5h*A>tCMdTOLDUz#Ss$8hPy6z6ZcEp#BpM%Bj|Q3CWw;I8HF0oLZ3by zuJSnT3!RhpFN10HA;}I{y2VP`kD$DOX1`x>1(}5Bv|(V>*%gtFzmFMTlZ5yDMV%Di z5oV_nkJ=n1qPL1pPZ54cm$2s_*G%ygAsHP2lStThaGvy-j5%)G@lu`X*rh#PLY=}C zfhw?KDbg(NJ4q{cAG!MK3Wx+EWjDUPK*CXP%9$HE;D1KM3psAm#-CMBfss&IbmIUC%E3&A5MB;K z`|eV!b_?XI30S)Ej?A#Cr~V)~j62~rhMhiQ_Aj)@l+-~qvj6llL;^~*{CZDfSb+h6 zp)KZK-^1&cafewEh)z2FsR9L1W>|+?Q2KS#XtFW*D?;fSqohKo zFF|5Gshd#E<}MI%t3cy9(DV!@*WB%pj@;N$-wlsVkfR>&E=*^Ya;{5WRf^@)W1l9d zZ=-VfvY@m&$l&_DUYb>TR$?=dY%EdS(Z2M`$9)YW?SC*d)&_ltP4*D|y^Ib=h@{03 zgYHh9Iph0s+z@W~QKAfzUf*(~ssscaJle-9V7ELS2W7K(q8o@A;!b+a7O)84AT0(a z2ba{9JF}Q@Z1$z1rSJ2>-^U7Sj)LD{a86?L8la|L1S!tG|2VLA@f_u1q}Aq@U&K=H zMgiuMz#kieBCFnGP5hVoy5F+^>d{B_^37#wZ?+z!smK9n5kESjzGW5GN1eaCrh)Q% z4MI^BU3xnywC^tK$vDf5d*2J`NM4ME;@7WrC%boP|KOWrn&6`R*W}VUSq!G@b6)uAXSa4G!QAG z2@udTgk5X}ae^ksFqDAy;<`NrK@$mvx_9V)ZtbE9`3H<+XVz^bJ(?eTYX>zG$xLNo z-N%}m%kdhds2KFH4*r7aZ67ojGMH^d5~mQfZM+!lFa=P6HZUl?42-c2fc~ez4i6Xp z^ABV&TOk$k^AxFdWe$C&5~{IXdYxX9$uIZ~3oNOZdt$+2Ygc9-<%%Ek<~EXnC;&N@(e89zug{<1HLJ6%REs z@pB|$>AKKfV##-z-k`#%9H~6#-&e6p@4>nNQ1g%`fy0UuiGBE4QZaDr zX@bfEREo0&HsJmwGX_q)jXrjplq5ZcINCkE3M!+>^(Ap_2HxrFePyO5ly6@s=U>1= z;=U)skOKK28DRkpe8^a>fG41U*ro}HcE_*0AKz%WO5(o*0xZz5wV!57AfhLN7NRA8 z{nB4zh}|5{pI+`}7PSsaf_%XVDW)BF?3c2VXw~0_c7V77&myvE2|((=#a6dS#9f?;3R&a;3kvgVKk!I#8G!j41k}L)mO$lT9X~X^{lNUYWuQ@@ z>_X$dBx#SyScXymf6WtnKvN>a%kL|*>sYG;RFK@hvIORYx&HwwI4n6DCOaO8|GohL z(!)g0vI8C`Ho(_X2ScF&fupXcnB=Jl)N}`@!t@_bue&}L4u&+T0x1?j(sdw^BejLJ z10w<{IsRx$x&oN*D0}}_Q}^!|CKS-z6Ss|mtF`(O*aW)hh~qZxSo>G6aKAr7PLU@d z;x?vc(5$Zd;FN!_Kll5+-^D4;A~itGQ$CBJnSk1IJ_?G&#W3pNjqXlGZpxMEp3fbR z)=LjeZ)JUigBW|zE)6JW+AClD9!7pJY}-u9P~^0$AV9H)I><7l-iD7fl3xeYrF^b) z{OunJmNX<-1UgDjC~%*28w)>Ku2|qxz{0vlV}hVZ{#Wy)*QDVsaQmuJ1#Wt#lzfN% zX8yBYm{w+qU>rJLZMlqHHx!VZ4s(rsKyL{K4OR^IfcS#q`mG5Po)yEDs$M;cVv0X(n(MH9wl8adF4Hi!7D*vV1@4008ltHa9R3R05=X7 z#a&P0xXB2rm}(dnfmAXOp3ZdAJ@#3u;~ZDPRU})-VaF*BNjlmic7a#8G5fmx2=ego zn_x}?v=(+E2dddg1_qwwaaIfQY!zO&KM@j8lzR7|=ySkXc*qsXh0KT*N#hragj%rF zJg^ohuC{Ai5aKM3i3>(pjf<(mCcZ#8NXTUBxNl+q6k!thXCWK8zBT=0(M%=zzL^VkwS9Wk?89m>nP*%}9H&hXfSv%tqjWkL5T?gM%BUKngUTd6pTrkd*+tHw|U?y{5Oi zk8Qp9ZqAe;2cVaWfYYx!w6MFxNMefvfHje;t^=~iky2p|VCoFaLMTD9jZf-iTU-!g z)!*_-5~a1_%Oqgq$y=_z0!qolj0q z|H0t!aW6AaTn~!H(soti@Ut05kQ+~g?Wg<-#Ir%_Tz#5}IH-zt#BP(0Dkog|t-g;&xbTOzh3b({5d zL`gG!hneIR*?vFfM2F zLJ~3nVGk8c&(VJtzBChEV#hU>?(89z4D=NY! zdX9A5f-e#Pk5ooE08YAnap$W)uJNS`&g&Qqr*K2OjkuAe4)5Hn^JqKCjZf>3P(*&fL>X33Vgx0ZDzGv0DNCMe^ z+&D%CI#-urDiJZ}Eet}!B;V=$!yVP#MZu*}$JG%E7PzU_SmL+Ovpw`%hC_O_-FyWA zOOVu8O>!LA!+OkhthEx>ZD!^;5Ze3bIrj51XAQyih3BAeP_%?du-x1t zY_fd+gxQHt^P?@dG`>#QDj|tF?X3_EQo_cWg>)hgjFmg|jg0|9@Ux!1Ro(!s2JT-6$g_!p9+&)y z{SOgEOVg&TWjK%Cn?Idc9_SYS6AUYzJ$2F!7?$3#n%?&4u-!CR5m0}1?p=on=ej`4r6O1Nn9152Vn$5&bNj=jgKdNEU7h^&B(r8# zTJo^W)qMYt~a-I zL*Ozu9@YV*o`&}SGDM~Wc8~)nW^?7}-R(F+!b@%hXzuTY2^*IYkOy}Gm;kT$-0$Tc z;X=p|RP3$FNEtkJ2@3Q>Uj7KtktA$VCSbF!0@_A8@}H3la0-YcklGHIQ3S^n9@BuF zSF_yFtO|N4(|_&XNjyDGdOMPjepCmA`Y%Gni%>d+38u%N>Sjls_G+vB`;iV2V47<$ zQXo5S*Hr@TX-x>bF7BVsk;({9WovhzS6KqL9Dj#_Zv0};4BuK$8;=Ubnkp-Xm9wdBh>er542zxXesG1R;?D8-D zfpz9004O$roCaXx1FTu>| zwLDL|zJoiR37GqWh+&09cuXq_JZ5@7%)*DL20x=WsSte!GBWEitE5eP9WYvlmhSI@ zjaWrBM` zot*)@!L5n-Y|(?%JOqR!0A|Ap@-k<$t1wF!^?`ml34@}Pf~82{Z)5OZfD&aXd_kp{ z0~A-_rnUH9fIaC9O$o;yAwUfw+?NjR-{6Ny(nVYJWSL{D!ToI zRvGAqbH2F6Q}2*7?{4k66`t%-2%ffxsSe8l#J0CgsN>;Pj0`mWW`#|>OPcl3E6{?R zC*Ay%_k6})Hkt05dzl%Qj6uO>545Pz^V5^^Y^&X-*p&KT6pHhPBVKgbJdFka6S|FX|;r9Hmgf*`y?V) znxU*WY>nbGn+G%Rfj=O;ERp}`@sDuB9k_=fy-b?O%7FQJL8hA%B7Z%4J!gvLCKNgo zpPV0;v4%yuk?p(4rG)T}Q9I5^l1y=fqrJ+@sU?!*RQ~8i0=934E0n+mA-0TFfuPFPK9BJ?v zhC8nHQ_rFiPGi%v`N400OOT@-AFQeR38PN()cT;DAE)GNRs65-!^p)2w2n-@dWAHU z#GVtIDsfH4q==y`^?0ke*eS=SyvR zY+hrY_O2T9OxD?TJUr zdr{iiLr;~9*KyM;Kamm-m>i7gf3`WRG5NJAx)FK+^#MTlWa?LannD_U{UOVAtOUf| z3t>~p)HO`uBmu2B09w0vmJ( zDa8!@2q5$P$Lvq>2hN)(is+dZ%4yq?Q3wMFh+>Xbl1e-2E<;Z6Nv$!IwBV^nMJ~6p zFrW0yp}Ggs(!%4A-o3wxyDrSpo@*khbBW_Px(qtoM#WSuR^w_#&VNTg$*uuQMy8in zHd6X%UD*WW`&RbjY0#0o)XPx@bSTfL!u_+ndK2LOra_X*D3C>V0#Bg=e)_#Hadi`N zD&xj|tka;u>7VM|4BAAE$1LmC8z5Q&V{W1SNO}tVRWVd>ObdK~FzDZm1+j_Qw+Fut z*T5KFA*rbi$zzw^*g#%TUTWVnA0-6LLNOxHL^Jb}sc_AMaClr|2}TuH4)e%rl|5CB z)zTckSCq_aIkAy-;TRxfC7PlfTS%`N>WPon z3V01A3+pQBmd0~9i?e{tX98X6O47*gKRci7+7qa~o;DV{_bXi38rsPA2G63slLMV0 zuwo{+66kGB)Iz`Gp^EH9l+i#!x*5f>d4Hi<;aaoqgSK|NY3Kxz#E+eL#v7&yF(NJS ze(Uui-(Ph*_E(S{mNm#a?1cC{SFbs~dr%fOpJ(t|fb6Faw43Q~tf!u0Gr&RJKXB2b zb)Vn)lKtqz-VRUzTn-n;*!&7a530uB=eJSuIt#on#}%d}mwUphF~1IKrRXmry6`^8 zhc6^cdlZ4dys~_8vjh}l?-Y)0UNV*W>n$n~>md z>>$hIn% z(uE&aImP~bZAYqQYHDvHdi(g}%5%InV{o6`LlZTw_wHZwdSM>JMBvSs2Bps@pF!vC z<*Yrbg-ai<#>CnH;h~k3Gs}EA`qJBJ(6Sc4OIK86O^U5eVMBZJ1}L2xyjMf-_T}}+ zog^_<-q+`|jrJctH9e@2D4tydQ_h7~pVlfpG62 z{ryqFDu8=-(^KnCn?3v$>a9zYU8TpNHvPts(oaX{n;eL)tMw8%0&pzoYBk$6zwN?2 zgtt`oZ!n0V_i*jAp9^CxjYFHMuI=O8zxO8q_nIolyr!288u4}Lck1zlIB`3K(sWf)u73ZdmCuixj1i8GF^06q z6c@o*kbYpA?---W5F&B~d;e0#%MlwK_8^4EXQB!bQRbOen&M&@zm4i|Cq6eF zqG!C$%7Rd$fj&wp$&U*MuK&cZtfL~Tr!RwCNG+yt@DXU<`YEd0IQ{kefdF&q!`S_a z<=5!ihh!A*uoS|LZ4U(QUUR=}@K;z=vdo~6aQ0i&@zTI>pa#+9y_yzTP!qE{$gv(0OWaLL6hxX8eJC2^$iI>pL_u#U!e0xKZg9<6;Z<h=ULRjyE z5ul7q8`k9TRP@Dwafx-@tPA5)*ViT$*e8%&JS;s_~8QVReL!}!4Ko00Wmx0EV6FhXl0)5V#NkjK}dkY&wX=ICx z=5S2l!}mh^8iH$2DCe(0LhGxXwYd;`O()?v8qGHqHTJF#(XJrVF&E?euYO!>uJbl} zL>EBcsqNP*NjUJ;d7gqG1vNsQncE*N%1dJ$!iqWgw-YYhCn}yBQ($|h-k5>*|9bd3 zk|NHy%-)umrB!rM+3P2#bB$FSffk56&G{{MrA`>N%b)4$m9CtL<;-aj;yG;#KbsNm zZ-DXcN6$Qhta}+IGn$J-@P7_s>C1<&Z@AC}sRTe3^DX{ZM;PD7tQ;BhmI$F_!4-6` zTpJ~w?6)jHb&g4VnOSP4S9%*siZ>=iNzfOQ^pqm;8e5I4l8?*sbWa!LfR9JD*Hr3w z7w$3^WJva|&hp)RW=WWOWbi=X%0K)vnm03{8ds62TZ{XA^0d^rc2Y1~M8=-;^F&=8 z=eXvM>Ep|Q`cL2&K%Tg@=Wzw(fdC)x}_$k&w2OG1TpCw?-NA)Nm zeY{a|4yj%WF~@7y@vTMx$?%=AeNO8*URkHB2*mdd*VFCE5%va&l1i53Kh3@qg#(N; zQ)1uhNnZag!X<@JYypZ$SRWBGL-N?Xw0h_^U`3jmo≧#mm(*7-2Pr&yx?ZPT(UU z160isd%jxyaGL)Yiqb~wgotq1-Ll7)hw*|PX!qEQIybTCW;I9^q406Iv=b5eM|UOq zs`quYsV!z%vf<~ohQktTWzOH1dMB&jB$Z5Nppf#=j|=twT5rWW{OSqF+IX+pL_2NT zc4U=yWU4+qaPbSn^3P*Fi#CiezsMR?lINR7tt1BCFS*}~var+!U|Un3CuMMyXW%wc zZY|F#;w(7UEf&1g-6;dDk-iq8_9kE2eJ37_~;# z`n}C}+(F#2@(V4+rt$u+Jl;VI5WYH&G+Mz~G(RY;jk06EiEKV|Xyed?MFy$m0+q~7uww?9joH64H z-;gGYq}#wceR9;?cjv)7Di|LY@nt`LnSiW4wFo)dJ!1(S@EFu3!}l%k26%+Bawpb- z=Q1<<1U%b=;{EtT+F9(v-?-M2AO_{vfSf-LEN)!53d@=wW3nhoN0yQD07`;HEXSOz zf2t!=D9kZ@2UupxTtEA`{IA9md113G>=C@8N)%_)iKmKf6iaXcZa09(>#gesJ{{#w z7R;xw`MtMpUVRU$O@j_ig2_bZqq{!BYd02(B`5apLLZ#icM;7FsQiuk1DCNk31+)% zz@a2xFoG5bev8`bi8Mm|Zsm0?Y`hq+ZS|o}n0h_K6n-{lpJ+G;o2HugYOoTL%Ih%(Xx z9!exXQQ@QPM0*Zm;7d(>5-C)k;YcBigwhD{SBXRCt1FuZd3-j~N{Y0mIHQOdb#9-m zjIbHG%x1cFH{`OdX$O8FoLV?Du&jWxquVnRi`E`*^ulaqJoFT})Jqua*B_g^%j9-O z^Ic|HF=*lpbbbKcBeb9mSX|g}5hMs%mRPhZ((g;hNu87xkct$o(F~^cHGp6$$=Gij z)$2BoWmpR0#ji~9GnPaC7?$hVYFESzWGg+N?(3So90f`nqHc6TU}3eG9iO1Vvupm0 zxUNdK%LaUtFc8h{DZiz%Q5zkd0`R|^yX_b8es++mwTOehO<{B9p6nm|5Kp72Jmt_= zZn82vU}o8YXBZXyoy{Q50F#0P+-0Umsh8F6eSPl!b+W3Lc|V~%5Op}B2Th7r!xRYB z4Al!a*46K~4)c7at2j6EQI>yPAt5NnOp)=OUI)MT{Ff|CX1vrB=vZ&T!~mdDU2dUL znirdY{JtAe=il1L>N>?EqF@vbM(ouZGKciZ>2|!aYhF&@qh_8UIp^VY4YG1Znaq>E zVRXFKl&^k(oeY?v2F1R%=^f2G|M1i@R8cBvFMWM>J@YH2P{+Uv_#0U;iEFMXjQ^nR z11B}fi%r(ZoQGHE?^z;(Nc_ZmuY5C@T`F`TO63i&s_tUeu*iw~&+@_VRZ{KZ+vRl) z>4&14pM>@;)sZJD`AA2hV4RSd^g1vDX3Dh-9~T}Rh6vy}+dmQbl1^8ilz8*t7C=bt z#`N~nv*TqXjtcql@;r`FT$iCLWA0KY)Kg@xF<?ky6V#|15F`)}Jq zztHS)^c_esn_rK+R;Iu3#k^F*F|N`1b;Izpd%1PmzX2@n|B?0H@mTh6{BT_%vM!tK zJu@qNMu=o3*&|AY%U1SCMpl`b$*N>$XJspjLiWtc-dWFas{8x8>xs!*bq`J=*;Q6S*}K^`@}Q(KV#K8nc|fTAbPpA~|n^ z8uvAdGual!7vkiZda#j!VcA5xMeKjgAAqeuR#mM3I2WU52H40L&fCcUn+7_K#m}D} zdai0uUsQTwuoLRV1>jSuOcQQhL6TXbWCICK?&fiAlx6F)qRLur9q7|t!oI(~xu_ks zE`g%Lbw8~NB8(y!5lvT*Vod;z%`>zIO{tdED1TXmFgo^vVc+?S_sjtGs2gCZp}a2V zlUK9bh+l^JmXZ5V_1kE~;{-`JJ46$QXq+8Ew6`UJ^tsC@u->@Nu-!cO;y-5zF4i?U zpQ!>MFb+SucvsHJZG{~EzaGnHY-bTSfNu~)Hrpv2@GS4ox2)TvkQObIh$`iT)&SE1 zXv^&Z6?0%ODrI0#vUuHEEK&5)ZN&uPBaN;`RfOj94I)Bv1W2Wqbgg)Pk7p=O)ZT>G zxC-Hum!D^iO;&ZQ=-b0&z<=fk3=771HK4Rf@Q*r2$p<(-s81>I2%eFguEUSvx!E!D z0g14vTS@vqffGPd7lSeVcKG>AdB9lu{=0~gi`G{z_#6Qs#(jAvw*5vP?`HszL-ymM z>G4KtVZoMaeD8!bQZD_XD8IJ1gLfpS9I`uIry=N_aRQ&!_~0L+2-M`yNLIT zi4Pl?h12n0mVmk@=v)&5qvBt5(&Y4rxECNKNyP&#WqGcv5uvI7E7EY~SE8yS+yGS{ zMDzuSq#01`tNd3JB9eVG1*HoKPUJlGMz<5e;)Th0&KM+y=lpx%n{{`pEsPN*(7R}T z64MCH1Fhf$$JaU^A0$cx%!C()gIe7GegrwDHf*G+MnwCaF>mX(Z2o%@PnwQF6cp)V zgz%LL;1iGRPXTokT)f?xx@LU#~I582eI@>uCF z==|U3tVMxYrU=f(x0pn)$^wICu`xHF|Nkes7zx^932ugNrwNvSV$rQOl&1U+#(!?|CB+kYak)D{Y<$-UbM007H2Rs6h>( zmm~AI^_LEYy2N6HjJ3-did19tJ+^~o;TXc8{}k28H@th!6Hay?z{87hlXi9;0Y3I* z#fV2P4f- zFxw*|$}90xkN?`qAf}7;Yn7GS{#3Ut7;K~FofF(P5!V*GT3kimcbHZ1H@*o={htQM z=UlGykSPEzaPyQwc*AYLH4Z90wqtP-rUu26^jDT!gE*Rz;!pFYA)oXyw{|9I%(HI;DZtw2 z@ArSucecEGWTt$ZRs)dr^1&22)`n0=8C_zi}p zn*U1@Rz!hth25p{{36Ba2tMiJc>0P}&CQ zi=m|SX9T6)DvJsP9S5rKx%oZ=v1eObBai#$TgH-6rKFqig;XJyUQSl!!eYmIge{|%mOPRCsHpn^RSm@G6r$n^siZCYXMb$M%Vk@D zzxoZmY-B}|h}plRsp9DeM;7P^X=^eLb)@;>I3oIaB@fG>^q#rI)%$RVBX4Qb|1?ER z$ygAnV(sE-wFK6Ax2{4+&9RM?`IF1tsn(OJ+WjaJo-ta)4{ndq!((`r?DKq%k3bfa zg;YU#M6jSqPb6`op^DN(Nib*sR>422ejxCwtkd;x*$sB~1Itk2@)1lSm`oi(@Bcn4 z)`=1g2B4V^5w^x(KkrCkJLmiX+H*t?glFoM@7y03OPt)M_wXSZw?8)F<999qW^6t^ zB`hJY{fwmJ1gF$PQ0{xc+xXS0{h#Yrg;T|7UR)tO`(zH}!*voKt>+XfdE$0av+&yllwe_hCD>K$qLe z;0}JWgi8lhp_yV8Qw@dA+?Q7XqY)^lUSamFSHc2k(4GCAm2waV^^MRy>b)*%e23fQ zoHe4)-JAO=ihOr~ea+$5UTX&!kGX5eOzGeES)nOMm4Zar!Ato4iT(P2K8V25lc*a+ zN{k#wY@88?R(e+e>UM3_VQWY!pwdIoz#f*}a3z=Y159fOD&L&m!q-;;(m;k64auPe zK0ey8)_G(hAl98GyD$G)q2J%gH_h;vjtToWILCzVBNpZ|I7oxA!p;&DUBQo~^%+*3 zCem16EfAd3*f2o6!{(oY5S{b@1TEVj`lFpV-Yb&eE&`L)Vur~S5V=OT$KfB8!pmFz zyo*-U?jknVHw885W4}t6Z2mQs2BARq`f6)=WC+N+3GI)qm`jxZv`T)wxDXwUUk1Fa zlSYe)ImPHd-JBm9ty#DRR`2&9DM7kGD1zO?%@>Vzb^0=a4MjM8QWrMx+7yFsI!Sot zE9oscDpi;AJ!n0C1Ai3<_ed}HHt!{?Zt$jt86t*JxRuIz-;)Zj@G{u}LY!X*aULJI zb+QB1$L8-%pIdcWOtT!zyDee}Llm~8&Z%nMg2u)oLD_1T6)lJ1WyF#^Fxf;0( zIZ-H3_EuTw?BwKR4vG73uA85Jiz{OZK8OEcS50oF2R!svKqMEzub0@tJPbj|cD3LRPZGHs zfdGeyu(nclMwzsjyHgIN0ZT9#&MFuyG#-^-h^6f*x{1RRTi#Fq>+yxh2q^ktn?9fu z(#p)P{}ldlQZyU02&o0@5%12W|{%v=r&z5}YD;Q#_RbN9B`2!@`_E|sS7B(@Y@D;O-_#7YdQ@;Tv{VSQw z;00Mozj3Lc;e(<9Dl3VQx3V>(KF402@dBpP1j*5V@5-n)lmyAU>N%jNmlQ|IlKx3X z{p9etczYmRR~Ct1;ijHPm^gTrm5!`c70AM@%9S&qj=g-H=QVl^$#2<*UJhPr^R*H3 z#bZvyMs&;Mdg~&ro)eUgRP_LH*L$^iekx<#R)fUY@4Ag`)q<|yo?Q9e48!>lQpA2N z7I^)$I1SBLIFAT0SmHYKLmNYI z#~DA8^~zaC__pEVJJ;3~6IvDJ{AMnArm^3zQk7$m1HtwA7dJmy(0Iw-yjlfa`yh`f8MMpK@>v9BE~LcKTF_GroZwRWN1Qb zikCwfXt3(ja-}!M`Uyz^^A?LxGp%`Mvc}mC2~$DQpdYD9g9u#ML6IS!*KdXGM0NS5 zow~d-C{)Uzcl!YjB6802v2)eBm(^q0d)D+2&}Ia*f)0bqg87+gva$S|NW6{_4adsf z%rA-KOkoxi#Hh+EJ12cNLe2uOX?FH<(I4{sA!7*nL~LU*;#_6dt%h~)t1j}^ixwGr z`tXy-K#Z1U|8*HS zPYY;%^|+qZxtSuM{nGyr$Xh4;CAVmD)~iatA|U}6Bgj7fby{WlYCM+#pUyI|bXM!% z09xpX8*#yS*o0(OUUO#qO!@PFH^Ez9{z7g+NG5u@SzeJ1GNhNNXS(axsqph-D)rg9 zbbF=&vtYx&Lgzyr)6Bv_69r}40`W&ckugAm$5s=BEx2jFLRlOh6>y6#g`B?ZhjRD9 zgkB1Dw5oMtwHuktLGBiEK9v74_mwud6pSkAu^)lbuQ8E(E*~?|sQ2-)OL@(;;JII7 z1>BZ?{OTSn9t9%Gp!=WPa7y~>6%x2K?=}RT8^fjVa2+Mt4gLhl5hA*XZQhh&eN7p< zE%#G;Q`{+QEbNP^x=PF`bo0 zQGjI5##{&ql2ga>-m=Q&qq@M0m@eJ}3@mi}(!;oK$Ku$>^G;;|37U)KV;FJ4Gvysf z!-%~5uTXy+U2|4CRbiGzbfzS4>kI#a$so@Fe5Xos!o`jAe*m(WsoYGB?$w5*;=A2; z=lr2v3dQfzq|DkVxiZH;djZ@J0}fMNV|3k?T$oBPaf%{Kh&L#zlXpU`=fo(!iV87 zH(S|vbqM1_O%7GcyQUcgaDk;S!5#cL z*u9ymmK}Bw;8AYm{4t?6L0^`tLqB)yi%IXwGHmPX^5{5#oTv*oX}B+qkkh|-c0uA7 zI1QW^bEV!EllGv4Lwsv}<~91X|?n-^Qkd{iQ--)e*jythBeIKg-ZLt6Im*JU&`T1VMdGq^azi!tgTw@>L@OcO{ zd*pv+se~_1F%tnbX3+Jws7lx~5H;1z4Vu>xkON!~|I?2po4w+^X?9+d2EO zKg}#r)OPT@UY`C$)$3LOdj3#UXOEY{6jA-z;E#zj67J|gPbnvdnmxqeyEJ5rDg9fcM@zoE!e!d*-su5ec9F^{>o%$<|dV2CiP^&Ge#w;*!at=8A&1&#r=M ztq`$y)=!$`Jbc2N1zjEU-zd>RSZ z$DehmWRIQYvF8wo^>_$$k=2)%TPzQbj^ii)93MQVeKt$magSV@?AW9BRO_J2Lesr> zoSt?2GP|o+7RTz2#o32-y;W|XiQT#)tM=!ZQ95|tk@m*tG<>Vb-r3atmc46lA&z0h zOdL`8#S&!uZ}?NHXKt3}~JRl%#JC05?5A$8~G z=s@LzcCeEY5qjy_nX`4l2PQ;lnupYB zqO7};ZD1u;_-meN{LQvh)~D6D@A#g<)JCk=7&RAPY=^ko-WHf}7v)m%ugg)rbB?}t zS^|l!bsJqobst-~-dyWAQB+oS=Sx(;>LB20_}^>Y0j&f=qag0mT*yo;g!ena;u%vnj$S~clS z(R~1m)xd0zHAcIm)NhXCv9k% zkXU)u;^JWLeoin4#_YZh8S`@E2e|bNvSENM^T`_GU$((aQ8&T#ZU;$91({);&OQ;H zW^>@LG|3=FmR|&IUYCkg=lF1wr>3)H?5fN1?og5O*T}S1xuW@L9Oj}DS1e&iUG`oB16uFY(kJiZ_?7fSUCW#YP`HYw^ zy>;eF$VRN~at-(WJF+<*U)T)2Yahn9OUFG3SX?UY)#y|a{IbH*^ci;}7#2Er4O8Ks z3(5UsFEN{R^eg9K7^jM(g2n=^hH~c$vp3a@$Lb;Y5mgq0+r9Lo_p#ViIu4yO&xiWr z2K(Y>s*VF|EsVI!4qs8P5I0>6>$78FLLq-NW3W^L9=ok>f0GQOq_EomagPLj8u^op ziz09q7t*rP!Ex}Cj~04CD;} zpS=DfUEsUS_$gk>bCQY`>xbN4L=88Km#_ZrG!r`PWE}7TpK#t3XFr9l6s#!!0-HuH zA+kZX8Uf=NB>7N??XFoyS^N|2A)vAsnD61mcNe|6QEbOOaj3ILchgHnO{oqq%%#!A zPqgco*mM)w%J}Y+U=k7?xLqt~qe}e{H#7_XtzSfvzs->N99~z)PT`TU*r^-yBEY>S zRf24S%tZ9zjiI@X@4Q->9o15QY;|WNF24vm$%8N#AUWf+2bZZW5HZ#|@k$OU`&P>0 zKA_)B0+>u(#f2rO;1=~bD?Z6^xy?9&4s}#vQ55<0ri}Jy?*Gqk6 zwuV>k4|COy<=&{8O(E&G zLD!U97E3vvN>t3ytnvr4lQY)ldy05W(|;q~Uks)e5w7die-A92S?9&RFSQfgXU}Y5 z|2vQ>sd7^CB$MZGq>}47FL$rYwUq>Yzv~m~EB2ghMgLnACzoSqN^o09=&)n9RB2>v zBHcZPRH3o7=+MMwvH+NgDL&x3cG~}B;hOxjJ$nBg~jyT*U@E7v@uif zowUsbzrI#UtA}~$GswK?cuxtPI&%^C6Q=Spgg6t-vPYldE_*GibBX%S&XYMB zUDyuGm90l)Yrd#`?unN$@%oTR7MyksKUoVg5cOOxT14{>1J349eucnSgmC@|ro-Vd zrk1v#JKFjIbOaEWBEjA>6R2d28-)I>NRx~|j0e4GoNf`&y%z}rt$8!v5(m6OA`~;8 z7vihixG1Nqoae(?J{+fESQf_G2?DC-=N|l&+oCl44f~N!vcSa*>~%T&6x@>xLx6$MV=S){mUZ-~#t1omHj+y98CRm0jaS#=u-giIy9 ztk@8Jf0^(z_+#H<7X4jW=w>`FnJ}?QUy9*aF8o9!0*q_Of%nx=PR@s*p?+F98OnQ2 zH)Q_+I_CHLWIG1K-JrstswNs~(b_4>EwI1UeJ97Wn{D5?qw?y&li)y~;l&8zW#U-> z6;7%|Bax|qQa{{5BgMm!#g1^%X71n>Va3O#(JPl{{oA)I{MYs)#!5W}AoHbf3}AdpK+2X^n2)eTf#W)9q6G(Enrq&XXna3KN z3HDHV(k>lzG@m@t*yWU2cm*BJrJ{53PFob|yQ#`bs~!QPiOh#rbgzTEq#48L>PJQ{ zaQdn7Vy>huCOTY(a%O93A+dw}=#DkU;=S&+m0?ryrj2%;!*S>I%Qwc=W!EjQRsP&v z&N9+I=eW{%DHk%OiZ(+%x9es}x>+*VD;QZcHwx^V^fS2#l*xDWwJRYwtyb|{`?=J^ z6j=LJMB2|dW(pwpI~SRaa;FUdD6Jf8Nd#B~Z*Zt2dVaaoKd@gUK(1yz<~i_dL2gr{ z8qKN{t0Bxdm{Q;p!^CacoxE`22PwK^RX7+vi#Z+J0J1sdmpPDe{5I|saV*wb0Imn_d9MNMf}+SZ35~z z3bFL@>44yXx-p7*GPztoGuEGGW^~MGe=0>3LB+SH)YMJfa}W_wtkLUTMV6fFP{*Wl zWdr&(TW2vkGE9@B1?gp`SagI?#lTl(OmoBOCO4@@?#zqwu|5xv(?LjG?xKV+DQqw5 zjq80nQ-vM$RW3spXhkGGjjk3)VPSELkyRwXE}=>!MoU?;nF8l2iJ~aMLw=)*uj@|y z1i$Oxfs^hlZZW_8gE_)6URfaV$qU?$cOP)8Zp-Scn=~!tE>l8=hv|DhC3^>8$Ctlsq!P9V)d`Q zl`YW8mrcb+C!01gt~FU=(g~MMUj9A7gD)3iN^^plY~Be>U(ceeHQlp=<2%6+7R+Hd z_ObNr#96Or<1$|kiIrKf5^K3Se_%5oS2jqP=G}24I{IWgSQuB0*(WC34@W_O{^toYoPMSqJa4M?p?u9V=1jIT zqi!%fQ{ETB&o(Ayqxk$r_X<)`<*+76#)-)prb%x|Z|T-tFI7J&v`DSRLrJMm+O`5c z$X@zWBZJzO<>Z_*p}5H~+^y)PT@(xF$x>xU}xai?AQy!ivO2)5nERg>^ z>F=xp3%~sj6XuWd)sJv8`0BbH-ZOiyV>1QcLr?DUs|I6WODLA1ZcL5w2y#|ww<`Ay zznWJ58mVj~3|Tz?O5=9J&biq=;LbZ1Tuk(&N#7IagilwoE0JfA{^8fD9nuMhSPB7- zOiR-q`=AxP{Lk+E*AWwnIE0Cfa1_~a95zxky0t(YkFBgkISrQXQhtrFYs5&|(Pn~W zLx|J+$cwv2$HBK5zQ8~$V3p)U{OhrtVCV~Gf1cwbLfP6E z7-xl;Y5x^!Z=Kx;Q@fMXI?5_8GjC0XJuhruF@fV$fBG!jV5V7|)5;amAS@)5%r=EP=&V>`?S*)Am(uEg4Z!q;)1wtDWK#F=>*iC+3ALM* zPL6j+_$lJOl-yR0GizNws0n|vtr!k23)-WIFg$nv1{t4Gs@2`jl+{-H06!+q^Nt(K zC9+%PZdSXkmK0Yv$HwcwkwE^e9WYE zN#3U*=UTz^_Jrv2hyk5D=2ev#CPDtx3su40PjXZf$>@W1OuKyMFBAzXgD(FSM(|$H z>i|X&7Yo6xvqfclv3s1%{&1g&`RFQc8fION%eR6#R!-%~y*8z!`VX163QU$Q6Lpp^ z95FF<$q`tS1`~*#<$B<{hnE+eTIDKz2pv*b zD^xEPKT63lpemOhbMm}fS$ z#Qp57-139qkq1~T53y8nt%N^X7ek|p{~SGu#*J)jSV8?!vANCD=ymz6$3y+Rr@S=% zTow;^_&T!~YhdfB;{XbPX2$ z#|PAS_}#}T|I}c(EY-?<9b?nrJ1p(T+YI^^(oj8W z+#DYhOau1iB@3IGN*4mk{Wmz~)EY&F_F z%0|e3rhh*0!P6npYoqz9`4Zr7HwDR0%6Z{2tFFLz{`Rs@W1Zy@mn zk;98<@ydN#e3JO#%kG6P!Rx)bMG-N{nVX?ikxC9e>E2q{8VIHkLD6cT<*v{>fb%L5 z6g+pFzr?h<6$q%DyII$yJi9ROeYWtqj2tZq0g0#R2CZ-KIs=l~ zTeP4^+Mv#yODxlyuib@%oXF*?q%jUelt@mpV`!G#OPb$h6#r{8-;j+r;MltmpTEep zmGa@~-*Wkc6r(_oa{*K^mVHMDPU@P0g>4}oo?0ZG!``^~$@;BnEW;BeWaO3IcCma8u)8|&TJOAM~A*XXY&f(`66xx;GrT^+-007Nb-tqSPOoY`+kvgcVJf@X?1)!m`59DbktFEyPuF@3 z3)XrVeI-UeRvc*+{wQ0qIP^oLmz(H*GCmb9TbJ~X(N5848~ya8g-bdNV;|ekJ#P$G zlIsVVZ*NZban`*Vc5F_^uyuy8!jw`MewC6Iea(y*zk?$-cEjC2SLRwJI5i@wj;}gI zvMIe*BR+Sd_J!vwrtaUK%2!)6zh>h`k%ZgR9(>`F*IrID#(VB~O!p?}?{T6b9k-?P zm!;ghqRRZQ=VOUFe|j{3x`C0n=Mnnt-${nc6s3{(>0AQ z$%Rkf)Q#FsF=G;U)t8Y3 z+g@L!t)>1v5f!d%e#PWDCkT*g!C?}4&pyMlY!+o2EpuAiliuaKRuKyA62n0~B{B2T zszVm1m|FMhs(dK7c6$>(XOlTS_meqZel+&p{1=t6sh*pawqitwtvAG{`}TVA3mU%_RDw+L`liQ!# zHz!gu2VKtpf0ez8mcw{?65R1Sls6R}QgU>tmbs;Fx$aOZI)GMMH77TG8xQ{xCT&VG zVgGD{i^6>cIy0kjCe`!x5Wi8aM0*{XP6|Pu{heXV%T-5FTC8pNjUMz8wgaQaR27M9 z)yxYrar1|Mi;^90z-p@EY}OPK@C$lg%zJ;KGDUT2r4L%Z&AiDmlWtb@7!0#m%ZHM#rt**hxUEA(WU z_%8i45eZM}9D^yWt6f(Oi8vbokg8U~77xdJ?jL^(9nJm!y=QT8E6!RR)PtF)pOs5X z=`Yd~EayNnPH4x#v4$gNPsjUrih0gif5wuN$T5RH+k7I*u(ixC#jqtkh75pAqP@l8 zGBJ^g;^DQ#szhlY2JU6zcLvNy#z@?je$hBp0I&a6_zq2oK44*N*8Y|63Yvd-`ek%70Kd|sqYcn(A;`D)G_6wEddVHRG75P2 z@`e;6%OTf}g6!^clU8qK)01Ap=Qz77KF1!JI|G%fm&iE9zeITKeNtWtOy~V%wbuiI$IBvyI%O zM7!@coz6G%$dl7A{J_a?LEpJ_x#8flPrg&9em<8Sihw~xzsXn1ju9D7Uz+(lV+c5Q zH6)v_%YN+JpnVTMkohH}4ihng-7dn3@0@h;ryuys-@|!*DEGApI4pymz*S%NA)%R?EN_Nenc|6WwDQZHQAFq6!B zahLR;B{jkgZreHGU3HQA-qzVQ^z8|$@kv-LDw=u2!Qm63=m0Q|C1h+9U*$SDYb6hu zE(iytZv|X5{TzQw(U`G|eTR@Ln2O{9Gs?1%fs+A6aQ?6yZmDXoVpMM(abjJyyJsYG=Ii>FslbkYi)ttg!0k~W-oDbh^w1^zQxlm z0T?A2SLr8WI%#+~qzciqPOC|0Aaamab{-%lH)JkdiUn>p{kreo6BF$ z#m}F!p4Jb`p1R2+;pXb8?{fb!F)w^QzBXJTcKo`5Q?7u(H|yc2?D_Wn=}o7}C@J=a zDqX~I#Qh?ju!|Ma)8N_14jMywHP9 znmyNi-LX(kx%1%=NjI=5*iPVTZD5&v8WYa^BZ$QdMd1CMgQue&HUwqDQR3I*%ZLKV ztXXDbvY$QPh;(1w;1x*IT4gw2n%5JT)0qB7cC7b;^FMqziMsGkJ)tOeBcoaCE58_< z;35op2-Sq6_%;$c`$S-^OgVVPTu79gb3jTkuEXTuD&+{jj`dyq#<|*qz-fn zO-Y|WGQaR%gH+!^k7A$SV7DI$y?kJGB&DI@c~`8Q$!HuwGBSQ!CXO;z)RL%Pr5P^B zrE1t?esWvNT&>3Ps>7E%`_-a+DlAsa^SirA0pA}UxH|iBy@!aQWy^;hJJP@L`Zk6H z7CiX!J-yF;9cT!k`NQuu7*4jghzW#EOIPvMit1!Kj*{B5qD z8~VD_an)~38Hfu;?wV>g33KvfFi8D&nPfIRN?TU6qtWq6wcOj9EK$G;4LKEX<>38t z&Jj=cw(^fYUApJgY)ZR+B_yZ>=xPk9ShIsKptp73YqKxc)7i+XV9$p2d94uzxFTQ* z=(be{F7^Zu3wc(&VH86>In!`9LWO@T-f$&HV*?^brESa+ahwkBHyoV6y=5jZJg?8< z43$d`bLOXnVH2}|iMDWq{hef?gH@g0)s$0YK4#||C|OR|igcXY{ncEjGG0~m8^HXY$$B)av7%N-DS{CX2w411^ylOi$<|#LY$^U`l}toJ=%SLW~M($*$iR|u6%#?`rZGRY7UM%MHvwgQ2zZus+xqd zAmi_0Q%oPnMOMg`U6=Y^Us><7Up;>Q9ZrZZ?w#;c0gYB6LBo$Zh1Zp2GsS<#eYcQ* zDa1LadU513`rsk`!>|{AWN7hkQL=;v`@gd_bjgqS+%oc9SSgIs zvCu4+KwuWt$3r$7jcR`3TvE|3tU`DL2j5E`7ezUmde@9v9y_2tD@3wGf{Vo^nZ!u& z-cmK$gof{aG+9~ol1lTFDGm*d8ooF#^roPD@Tq$2u5+1@tkj;L!W->h5Ai2*v%Kmz z@wo8IfPD1n)N}vvmDC=82_QSHUSH3pn#=_NL4W5g7T<(E=*}ylq~w`#P$)CwJRVuu zz$7MmMgUq;j~FGfMjJ%e+?Dv#Ber%dc>G_PpIllo(&@ab#3k7w?T!yRTUc&FOKyOV|Z^7R77u7F69|C@a!W3*nBbMe+0^NU1b?MH`{O2=O|JvWB@N<(VCD*S15 zw^=~Ls(fVl(wZew>}}raQZIp>Qus0 zg@L>KaZ}C83KuBjCq>#ZEFYzU|IX#n%sPd7Uy3hU{9ICLAv$;Q5 zqNGs-lM_dcEdSsgE1NBaH`OE5M!m@Tud9`X%d`# zLIQ=!mgy$y_U{2@g7@~U1kpYN=1&s=crI$PE2voyHP@8+(QpZeljO27IE>gk@O|o; z@$N=r3T3%}pw%bHW4WmLIB(KTCiFC2Bw(5X<_gS{jzg6WfezHHK zt+ke-u_}#ER$EX)g6k#h;Dfu;{itaTfA+jAx<8r$=d`~-%ar$a3oY~@S`~5PT15M) zkZtx0Vm!9PirzfQkUCD>Hmq|@Os+;GM%iG*3F7v-BEd;fcW~!diB}4z&RHK?x(dwg zH@8c&DU9P>ZTI)BBqA3H8VOO+`13{DdtYAK)X6!A7G8>7x%X!a5||$-p+ag|h375j z1qn^Z=3}%5WlEnuU@R}wRjSi67Wpm!1ln-xNGENU1J?ukS@Y_8?&$cDVu#2Xx32-j zGWXueFHD>vmMYBS$@(Dk;wWi;@7KUVKJUA<2@@p=U2SWY!HIHxW^5a8RZHcC%kh2X zQG>Vc*+v8C)|ETRdV+pv@|PBM(#L z+P+i^C$<>=is+*=yY}L&@>L~mon>i;# z9Roo&BiUxjg8)A-EQ}Sl1wYQ31CBX{N~<3cRW);iV3+5?PHCTxkC7&ZeP#@a>Vo|; zlxkzn+T$FGhI^yDk!>IT#3(0C(Vx+x>rx|dYz<3s9c<5*R?OW2+al-hr^&K7L1aqg zHM|*P9QSX;be6cJDBo^<7^_sr^ghjpORcK0P*W8D5U>WkMEcW*im!#HM!NoYUcE zR7SKEx7|RFU^(Rc=$3Qc^$#iS57WLZN+nL+YPyVsS-;^QDI)p6Y^NrAuE;dAQ(=FG z$S<^}^%oKL4>k%^g#g6D4SXjWzR7t#HMt{yu*@GNGF;B33et&Lg`b<;>;xyiwTs%pK8Rjd;F2x4$QgMa>^4w)OE0xAY6V_!fwQP-Vz zU*P2Jio)@OC_8F8M!SlMpo`mm{Qn?rO**Wr5p*`&`>nDMEpLc#eV@EnYQT-c!(F{x zB(TbDd&H@ofo+FVI~Qz`Qh2sJs3Uc^70)L*J$MN%Sm}IMh*^t?$FbGLf+b?hvXHL- zDJkkz`m+vZ2a*{Hq3CyENgXL%4I%v&BxpZJ=KAzTh#33=Qu)pqcZp!+1;3+U20?1S zKyNOB?625^B>LYgWeY#vCK?AgduQ&vGc3>l38tpc97zwKx0{#RN0W{3$Z&`3N(R;| z6@p8})#Nxtj+yMMbXiol-9_6e69gz4T9a<73mxVJRnu}B>krM7XUb%OLRJAr5JNG) z*2Ml!?B8I=MTg={#`r&4Yo@}_k-x91MHURII)6sabq5_zE(35iQJfN@mJ^#mv!pJm zC{f~i4-1^p%ex_S;O|!=8u})QPb@_P#)T0}xIHy37 zJL(CzDZYSaAp6fs!t1LERspJX5p*Z)D%`Is~;FlXL9 z3fRzOB64J5Y?7+Qk0v7*Qu_TuMKe$>%85&1nCz$=R3q^i7PwFnmP-uFX5tLvRGfk^ z!9Hul!IKwWNNRa+tNt(mk%9p#2gmLiER7d}8iy&xes}TTm)CX$1$&b&Le^#RrDNrB zqXWpupEl>!wL<x0Wlr|`Yx0+EtKlOnDk>~z^71w94b zA8|BgrS5&ZpU4GT(a7w^$H_M+p;Giqz%tcEU3BQ7NQ&9t5KIP|fj-OZ|F%5KS@MktXBRwpW@pt9QzodFQ(}l- z%pY;n=i!PT!CCsu{FKhZ_>$2E5|7DyO@arbCz%1c|67EBWc{dgWxII69yM*gGF(~m z6T+cB!wc8Bzbau8LCSSqbDA!oQGrE9@+_;Oq{dg^6w-rT;5!ZH<#J?Uag&mn(wOd0 zVwzS|;B(rTm~kCRL%qJ%0HqwcNBlxLSAjiFf(1KCkU6gFX{CGS-yz7ALm|u7U^FVZ z#B@o7|G4I2&dGmb!TA;3za_ApAt|5G?SYavI*N%~w(G@x?!Pq&8?{U<`O*A2cEuxo zwt%F&$qqqZWIsp#L^8d8LyUKHdEirANEQ4r1Ag8{#COY0&)R#aBcV@X_lyZIn)=M# zngy`Km-MYRvIkEtT z2n3@Z>{Xe13(2;qO8>&TqA~sH`|)gFD;fDsj;-5F2W!Xhgt*J<$k3H<)L#FRaq4#2 z4{E-`|AD44>-`l4)@_FfB}Y!4!icxO5Lx7f)3rHAMPuH-zzcK=>}O4WWQx&ANxLXG5W21_@Bskkw`lQuDqRv758+X45>nj7GDs=Pi5fZwg&7{1EEJ z|KJx$P(I9mL^dSBRtQ@aH*x`Rn%VPL&9PL!b(m_gh5*pD%W4&Ty@zC!Mh(2rXKpe4 zpo3Uy8nA!56!9#&?_&pWK-RD@tgXrZdFRFTjzOuct@-2!J*R{T@}kM`urSgz1(|>bJ+QEYsj}dRbV-a7wQh2Re1cCrl2YA9jLhf^R2R16E0?N)7Kh+ZJkGD4iAi_#$G9rF795PAI= z=3C^xN%8G$J~U!d)S8y{dzW9;@%iPY&dXS_2N3S2eV<(R4wy+E?ljn%gQYMVpf=Tb z`DF{bNTXrc+uSGL>gC#tC^jyFEz4VKxnk-wvQ!qbM9pgVe$no4ld(L1UpCnB8@4In zh^AuhRQXiY80{tV)vdp^@6R{>f^^EnJLPqUUuwPN-?l!%r#eq3-RPI3mnPTbdMB#y z!dsPDBZQr>hPb=SU-}?Dlc8GHq)QRb7bI>P;+&wrdOcYcCw%_Vw|(|9xOjTT=5Bd`WtS8l##OyT5!bQ`%Z(t zO29!%Q4MT_Tt#qR)yU&48J|zwO1-&_ttv&Z0L$y8rMSPDxKChjK=<$x%uI>ko0ZYK zpnW(3szSXQa>ImQu;cl>Jo#i9l*B&Z-^9PXeo*d?<@i3b<(*}Ar_9l=WiR9~@@hW< z^`7vB{PFGXxb(IVKlGnffmHYZhpX#=r@DRLoQ^Uo$BMF!y-CU{8Bul-A}cadw#cZA z$jmOgA(UD6Dk~HjrIeXXW<+HF?+5kv`+q*~r_cMo9mjV(&vW1Rbzk>&5t~F-GTdM_ zrW+o8V?Ng@;B>w?yC;w|CpbeE#ozEHEUq1Ktt}w)+WqA97C4L^*FAz(E+v&=A6m|( zj?3gr8t2h@$*qhz5zGwo5LQ4bF?!teh6%D1riVwh^|rH=iz z-T|X!Dq;ytyP{3{uiO!7L$)cOZkgmZ5s1n?Ln0acFi4tKqLg-NWyo1J>GW<)%=M+# zVuMDL+cZ2GEZIyvp8`Oef|CK~2Dznb@jx4g!v`g=Kc{&%C(-#^6i1=pkV7#v zFdQAKqMPR|Fgy(wpVvh9wP|aUwwJT5)dxDE3ZyT@z+DuAjEN(T#M!-qW^mCQSJL%J z5VY|S#du#D{v~szV8Wj{OV{LClFJc)Cimxz^84*Vg7*)yF{ODzx;D^Gf2&?^420ba zM%IUzHxI*_ekK7%dd=`67&E_~KJG?FIo;}YrKFFJ90pk~q)`b<_$LFUo`aL~r{Ae2 zf9dl@`lD(5h=1PrgnIkQ!JjM`l*tWlEPKs=QiEX+MMb5-WQy;*59gB;-f=|4d62{N;MOwk|r zdD!Deq35GVH@>+=NKNu`-n|L~9x^5rC1B{(4DF*Svgq_z1O~d)Kg*9nMzXYgetJ72 zVa0oO_#)?uq^plVn!YRF1ljA$U^#n0|Hn%_iLfT5CpNsS*ehsCrtk&GBw2v+!{R(Z zSGM+zKW(H@Qu_(SC+^iM=3;yfF4DvN-f7c^XzoAlP{$F)&Fg?7#-2Yb)sA1p(;W1E z(}s%Lv7z;t@r}7gc{3kz3sByq&pxf_1m1gH22HjjFolsdPu`+2mgp2}7&7?@hLx`{ z`s#WmEr7;ZkDy-MH;^O|?KLcy;U4%FJp;06QwEN`k+&%bHPU36uYhWi!0t22@N0Us zvM;tvGDYm)$I>QNy(?M%<7BNm z8)D`I7hXxG=7pS?f>9^ofWa+UkaA5iWBjnM+{V#w?zJ-fmY!It#N)>D!=n@%P_>$G zG~M3RB|KdM>+sJ7GjpJ4a75$o*wws(g znb;1L%kVJfiNonZy>eD*G}QKJOs#OcEPaO|+&w_jX3dk#;kBy)sT(nx;f{e;U)|cG z*MR8p+n@XI97heLl6LI_Le$=H0b5y{jL4KbKf%2Q^nxzt-6b{yPHU3 zLo@WnIE`eo&VhMhJ^=lC!uvy2S`3N&+4%4C`~ZOsG2obo3G7*7>Y8eM#-do$Fb0`I zvEBN~U!TwC4%H$8Jr4a;%G{e#r|XT;R08zix+DU)oP(|l8(j^M1-&GmJ)dc52Tlx=1;ySlFCFRmaZuMxdWCXC{7HDiSbleX zbGfrS6>lTclf0dGj=w=NTfSIzXa5l}YHH|J&an6!C$|4Zj}834#bRlVT&tVN+9b;7 z?q@No5?S5@I8yIarqA|?@$+64+=gXl$m(5INethGoB~^s3VpIpZu9F?_1}DXuwo>!6d}-D! zPVe@7r@ON33Wv%>FckIO-Y+vi8bRmiX(Idhvf)S-YeN7nE$hRW?Zzn1=FT8_v`6ck zxZB`&BDIkmJx0;b0Q5Nexi=#g045ZVurLgqfVevH#fIZ1&X;$#fqgJgXYHrBUCQAh zX3T>(`;YV6cD+7@F`;;6axa#T!C;s3u4?+P#IoHKEx#isN=MZk<@K<3;UDbAe~6PT z=bT9?uSp_tz~7=R7HrF3)&Px34?|Y$lZ_HT?`}Li-fr**l@steDqm|RlysripSy8Tal{# zk?Y^j0jGvZmc)Dm%8Rh}C!kz&cddTThx*dg9d0vFlAIIean_SuE!Q%l4hgpa(Po#% zF|d$+qKB7pOg?DfkwdKclmfZ<#lfIeQXiUBYEdLEx#SZLSe$ON&v6XOvwySz9N**Ckk{R`o!)+Pz;r=Og`i@iSwtbN0&92haI#@550qZ{c6 z1=?x&{bc=6cKQb^_w7P!;JdeuoxjE^Q&v=i`%PNX*h_L;gOv50Kdw#e-EFsA0 z@slYeJ{xUD%=24g5>riFLm{m=FzhQBkeV3$pi<5C;dSd2pI|O~$L`2TGKzh>)jI#9C?XH)uwzA^IAtbwr)JY7?LG$+@m8aC^ z+PpADC?8+D)rs2`9Zf>y#k@VXifW^6DfjDp`@}w;fQdjwVmX{NOJ4Hp^xdBPr5nCg zR!y>vw&B8wz4VJ{L=o~Qgy!j*_KtmaLZL#-K>4(D^*XrlpV z)uj{F1c%2%s6XQH0@M-N*QElIaFP-+G0=Ud>xr{Zn!60fM|oq&mNlfb-@e5`lXQ;{ zBa;u6P{nW4rq^7rk~R+tS0j=k^yeSNmPb}TbYn15^d9bsU;71;bd}8jj_yluP#mKX z((Gl9Q177NG%Dpk6);xIdGoY2rTWLar&`=(SHw(}`?--e-Ero#pUp3%%o?A#9bnNj zXP^;q7ny@izN;;Ls}w-(et2Gi$#&K3S-@BBV41zqGxT(cXO`Yl?mxLV`L}m4h3#pq z&msBC;`{vA+MVu|x5M7H5f&fs=Tzw5eV2F-7lW@$fMEcMtmH{w^SujrCU#{fM$^}i zm?H9UP0?SM=ZKVuGp?NJ5Dt*MTX(Vq7a`e7-a;{BY_Lk~xb~Qt^+?cI1@()tH20+b zqPq3EURn^3v}Ww685HX}BPl@}P@VX@ML^|7(5VaHH!%G=V8KQFG0%VL9sVnEKb_LW zQPtZ+V;2*&nE!0to$XtTg+>(r8?tuDHq>85d>P({cIqhD|7`En5rwMK`0$F;(yooc z?nmD`*dB?l57RLSugQgxIsIHri?AThzF(bkKSP9~lRQd>5xh2_Cb}|JD_yaw>dI!k zHiz_;A_BVWC9>`LLk|jRb}~nhbW@2Bhp&)SQ()+%an1%1YVQ^hS8&(d2sHI*GhF^k z)WRRY6M)oJRp%w%4*4IaNv5UMw=GSrZYOzmh*m(q;II=$gD5%5o|L~>?Z|nCOxzuc zJ&f1G8P^moDrZGOf4G?Eo36TXRo(80L*6*7B~KBH{Xy6XF-hc`F^#1beF!=xEnIWN zedqzDu-y3|s{Ns6wf8R=S)pbcrB6y;?TwWAr&3tK%Pn5u-sbq!%R7~N)=XlozQHM&>W1d96>q1{z15D=YR0cl#!oo= zd}uNhV<3O_B1&DdLArp3v^su6&fc&BG?Bk(vIk&bzk?p-V z%+Bu->{Sc-p3#bsH`Lzc)TT`G7g8)wY5BgPv!YrD zDnVr_tnACdCG|BtteyUj%THq_-w+M5d~5ROpxD}*c+xws%Zq^Le0);JmzdK3*bMo? zWrIosJig|aKv0zx$=9QrXVeo!1%g<$@9tnP@N$8gRhhp0I%voeD(rk01{(gIF4RsI zcFqB7AIz0c)Vk@$Bdb9Npf_*G{PUe7+045N8m!m$eAh57_x{e$a9ZKCBK{%1L81!J z|7N|OoZ*akWr_pY9t^$lXCJSA&^98sEBZJFB1Uhq>QP*0LszR3D@HVHOBXCt80i5u zl;xK3O$u1sm72GSAZ^32j?LQsf%TNf>J7V-4~|W|ISGyI!tsil)u{b@pK-|x1w0pp zCc-SIl4s~!J7B|DfGS(jUSH-CwgJiEw`jN%O?uKIM|V_48z(KCFkhX~6PHQ#Sa03m zcCtwDjoaPDTBo0mH0mP;`Kh5e&yMU}(vZJ5Zot*UJV;WxR`z>d+>b@K@n{LTqtK8f$1f5e zxqmmIB%u)-hZxv5LO1QZ;6a!|a`+_f1CyN+!Q}LaM9fy3BgfO}?bXlwA zSoi99Q@og+Z6BwJn|R4q=ahB`clXV?O#F0j57r8^zP>pZ=BdlYf8*i%`Lz=F@CW2? z(-A>UAV`gZi)A>uFY4$1F?Jt!R=FCt=L#QmXcwM+$-WqO>yFESmhZVSIC~B+_1<{& z+VdqNCJRFUry0tbr+XOeWxBqSj8m1;vQIGQgeEfA`e|SXdAkB}fvfTV9F#x(2w3M~ zj_2qiy@2sVw)ae$n~Y$rvh3fx zI!1H|8M!$y6|Q%xUE27y(=|DC>QnHApZaA8ysr2yoFTRn?YlYOu`;U)%4e6y)43;o zsnm(>@;+KJ=&4JuUoT$+)d=2*Z%sC9u+(+A5?ooa@~4Ou1cmDKxbCk@oQ7Vjjk{=Q z7>@7>5o?gVHywgRifcju{Os!=ZhI_yhtjHY?bcT)1{7pK!#iQ5uvxVGcjo!{v$f>H zB;zf>&W(L}qE`9ko9f-U%96R@wd4egECIUtu?U*JZZ;l0^y3uiF-;qmhb+^KIVvqy z#JOLAMKo?_WP0+xQ*&s^%Iu=}j7-2A*Qe}8_BRu=?a7jD4akuiOb}irO(&AEbeEWub5} zP*dB&eed)(-38h*cuXg2zoz)CN9);xiP=Nx2LCUknWF0Nq_t>Va3x}0_6PQy z_~AL+qnk|u;<};v)Z<@pGU083<^fM}+BWdQNG1n#vWzRgl}VXL<}}~GzgN20#x>}9 zTH6EtyR^?1;))Mn5liOu71!83C{TNe*azn!1;Cg2fk%y{X@_@Tg=nfH7ef03Pj-O+ za2S?H2=^_*y*Uh|mM;?0t$XDafkl$1bJx5^;-docndyMv{?hhJ&7-v@-zF&Y1Af~_ z|2&_N+I{AHsd`T7=pm2W&sFn$-ciz!-@*&@waZ?7vx!~eT1k3jAy|-p^1odo(?Z#A zH}By?tESj6{p?~(CuyeXPJr5aA@J|1l>No)FczHnVY>2jQsI#?gd@&BQ>3BZN;IEe z!4T^z5-MC3Xp^X;D%HBq=)AVZ=H*H%Tnona%|;&6<}JXKNb!}_hTKu^v>m>H*%3L0YZOdLvfrn-oxYsRdC!$4LgQ zQb%*XQMmx^Vp~H{CygR4-3{fU*#SX=D}LUpA{2fszQ-rC^7qb|1H|>D6wsj!a-Y$E zQs!3T1rTAcQ2fY((>K2UQNc~eL;J%2mO51w{pJP?k;OskS%+D60Ec zftOUay_c2&$P+ZT$ZBN}o5c}~SK7{We|<%llH+KrG`C+5AW$bjKf1(1mDW=OLaU$~ zZZdyqNK+wRL|%xLBH&Dacz^a|ZSPNqc-9rCW0VT4Kh~^Cl{`>VP`s-0Z)7@7s7W$} zRdF{mnmM<9r2YMKn3m9pn&TA7X#z8vbJU!$8A#_}WoU1|Jw5zk5MEq8Fp3(|PT~zU zCR!=bN+;lXt*EjHE#der$FU5p2D}`?CbMxoi4}a4G`x|!km}MX&;<&3Y9^RHU&S$` z5S|Z^Ie|$DzHY@KRML+T0f{2l1dc*$&G5G2-2Si?q)NTvaF+M<|6I5?7#79D5!ZKf zc_hZ7#+O8N-MOMz>DP`Rz)q23%2M^72s~dzp*fg|-gNHKa_(*(Sxt?HF2{h!1SmG% z5Tjc&^q@8@<&*kxfnL*sN6`47prjMA2tm8p`aSr;TEXF{>r4H4*o-(l`ChaQ_7-O^ z^|cavLug)op_qm$?gc8`%tN{=91Q z1%1F&kKnz^k<&Sx=hl}G@ataMScJ~$xp)@=EQU`f>o96Ib(#hXSYP}AI8Bde@Q)Z! z^zU9qZoG(8!y|U%*F^hhPd~ZG?(axUckH~N{PDjFAwVXS%5|f3tm#ns2)Mzk6BX>|;RE7KxON`n9P~qV=-Lkm2F_7v z-W8>v!DYXO9=#C{|`sap2&K&@$R4fzUe{# zbIB@RV}ppwjl^Zrp*iulStUX?E?5e)vA^r;(h{VcI>Jw*;-h+C`R;cjhsM7RUt(!P5Cq_a;E6 ztNshu@i)wisKb~J_F#;ODlXDCTQURNm$t0z)z=};ehVZ;Mv-785N<3gL=}GR5aoAP zwhy~K^zP$9p|_iM9oPh6KGshU6$-;+7qv*{*tyAY>a89v3`l>lh7e*g+h46T{q!EJ z&!Dg>&=hF`c80Cu^6!|+wYpbudG4!Ef0xPj99V+kVsK^Pn2|gHT$T?Z;ASzf;>w4s zPy=64$kgAcCP^Oq#(9zRznX(c>C#k3?kF7OM`6PjBJax5ejhT@yriqq|5YupsD|mi zK{U22Jma2a3}E)xE3}P&HG84;UiB$w=eq4d0XE^0=Sss={*Q`L=(tkj(7WgesDga% zilLpZ2AqKNZ7M=G1@{h6a}oALxQYZ&vq=;6aSSN7s|8oIgFNFK`qNu0A9}EF6ejZj zy9nx|-Y`B!$8b-{@coZuWV`dYJF|Y94z#oCWpb!s23HIzA6h4=wu%ph7`(`tfXtgD zl67~+;PwU&3a$cPiEUq*nLA7;9>OYb?e1BUo6xcut))6!5bV0T>q&qd_U4v@(fWof zbZ;&PgU%qeh)Ioa$5@^Plq|NDVMPF3uclFqfBUcR&U!8ctN0?6%sf^3{#4&M;zJ6X zo=e#7pltto`NuUR{4jwU6p>T8;ri$PjDz0Mk-d+T>}yEUfNy94 zJ6Am;6&iC-OTkIR#+|aQfDs@>rRisACH=XWb0qz8N)z4SJpO(Ps#yzQH2ZaF6{`IC zz}6=nBdeftwh-A z01Lt@-6{9Zz(7CRC%AMvvP zw?$}_E-lWtCM+^a#)4dPEEwgs2SwVeY5sT8Zj+|`30f3G?jHlD`s|dZMh|+QGxIOL zY6S8F@=#0p#J_D9PH8T=Dg(Ur&)V9~0CSa})obU#6#@e0`mD>s;kO?h>~ zCZk~cQ3z7SirU1N!Vqc`j5MhNnazf`*dXV{#P*pG?zyEH1sx{OIG`8CJq+(|22x`pTE&0DE1~5$&f)8{qSjQMR1|~+3u4+B| zEJ+fnS(6H~9NRHm0)qj>mSAk!(3G)5kn18exi&7T`n72*?%N)%1B+B2X7|uf2##Rg?`UXzT$I`$w z^5BECUa~+qz5K3=wjbE8{1RCYk3aNis_pMfOR&9a7zvurjfokLc(=a`o(vMXWde(A zwB0e|U0Rp_D@LAQ!T_c!ig@6ipnjX3FKqe)tUI*qv#-1U_bE6J-<`#r$YhrS zy1=@1{P3E!-_;}m;ES3EXEkmIFPyg*L2`-%5OhJzB_ZnU$xbxpH~2?T&WhqYkmdBi z%rg#m^TiZMt6em|!u9X3xQwFp=E=&5jLatBV3^}L7+pF{{2f&V;yTu4H-6aO20O%j#}Q>GkhhHk zI%Z(lKg$bkDhvkH>oZG1n!6dI-_<%rko&mcBT%S|4T-=jz5a8IHU?I~Te1QtIXlEG znTIx8C-YTcLpez>8v0Xgt>$-P7^kt8V}Q%#%CS#WwCm`P=u>GK|M%M7kzq;>ods8n z&%|wmtOS+QMcpAAMy$}8FAl+-}%Zy#-!8Vq3jStOOQ!)bOQUYPvadIIQhcePj( zY>M<;_9tup8?8*SxI29iQ+N9}0$E&FgL1s*pJ=zW!Rk+}e1Dn-$7&kbDvjQyx5ZIX zC7`>6_5+dp*I@BNh!ZE(nj=Ft)&pD;-54ov%Nl!_yd}qrC+ITFC zPL4Y3Yq|aNe7G3NRJa) zZg&+2i}qeWe;$g<<8%*3PG$VNn}Mv1I0lT7prL&@AMlY_VAt{7vSm=e_)iF7J=cM) z_&rDl#VOVK?2E_l&>^>CM7tAogq^PtsztQ(ie-LcQCg4>b!|QvYbAiWS{#@YbuSOA ze0Z>ZRem!4Xpodg&ZWzW>r;j{9}0VNR~~fzJ?`)rHC6>XkDq?Z~4fl;>_MQFz@A#NC#hW z>`ST&Er?Jh)Y4zO_dk1%nt;%E8H}lq0zNJhV#tFEJxcae74_AfOi#uaiws@MT&Axg zO1y35x{K3%vsePG^)s$mnjP5Z8Ws_W!Et~YkqTm=4*P~AtnU`!|E=Y`dsq)2|DFBi z`0w7L`Ih4efEvU?%SmCTzSKELi~hg0hw~;TZUVVYWZehBPB0CGjnbg>p@`044T^)c zUUze`-DT_Z-ifTUFg&0q+Xf^f3$W%eahm%X?gU)+=5+UrBioNXj;dwV9k_8{eeRP2XzS*~vE0@3`k0S@()LR7USpXB3%-kU6yu%H9HGj; zKCUTZ-!J0?-TKk9Y+74n@jnsn9`RwUCbXJyhFC9C6^^G;$d4jj}HAhKK`H zqZtVbx$NcLlbOy-Lx3vte)bHK?MLjV)(`Knz|Ls!LG7Cs@au?$nF$(~r|pvHnz;!u z$iTQ1PLO>ynp2_hqFkJ6Db93wxDy^dtG07BTVaWqIBRu;Rs^qDxKTqU6w4NNqHUwO zg-qe!Z2~FFF*I{`1d9G=%N)P=y`OK)Z;|Btn(=&vnhssqly8Q_&V?zV(Uvc2LjWg< zf}dW}7S~!$tkubRqT)w%wNH4FhtPSgU^8#DDpw z7jZ4y;TtmslR>2x&4*T(%68uv)Mz{M?;Z)lANJh1$tMp6Dyw;g(yq##;|(PQ!b4I zD${g*x~pB1orS4x#WhT7k^Aw344!`%_#81hQcw$Q6k2U!V!tNLd2LF~Up4=y3cQsd zpuy8?!H5hjMCFl%&kPGKI< z!vttXZJh&a({`Tk{U|5>f)+#$Ts(ObG%ydzE3JL3gcX;6{@$T9L%gaaBsqMK z(=eRv8KzoDN7nJs8I74saq*dUdktnL@e_un~4q|y&F1x2kKV~~pRUdXY28-##( zR6fvJC$zs@L%HqkM6UKfS}~36hvD#;hZ5@$|3JC(B%+O;A3fQ%Q!zPXh-Dp~uYxnz zUe69Fc}7J!-pcM_G(*|!RqP;}G>mN>ycV$rZ$1IKeqSXy8Qs4>+2D--Hv{Ij$H0%K zZOw>%92WQm^n~;({j8n*dHX;zOQ-^f#hMews9cRo-wT$whVCXzq&`9f@ z0#u)9YEGL!nNF!%KT?1Ou5{KxXiF`_O*DbJ=`_JI1jKCr#L`<6ND|ij1j|cEy7fPS zO=moeY`i0lPzN<_#PCBxqp`>(4vxR<&tK4R0wSJ|Lg*6(fv#i74c(6Kep!^xzbP>_ zXvveKJw2;W>9C3*)^pS6FYh26+gtGLC5X0Y;O-!4$K>8JssY zx)lt+&7aY<($ikD)04ZzQnHIW>wDZMvn>)XDcfH9`~#?fj~Ti9V4-B|M}DhlD$IYj z0#6K<=r`Un$RU~$y*SeY%y&AALlm1`pm!Y$^KG%9<(B$68qFS%Mxj&?r6sy1ZO{n0 zuX(Ub_aC?g?@5xSn_CkU`OtlW=M%U$ta?O#KJ>rmv7&rG#JIz7f)vW7K#>^>RaYD^ z;uFBz7p&>JIlI)}D?FBio{ZL$-_G_srzQA;BuY40UlkP-S66D`u>#(Q1z=lZ0F2}( zr+^=K94wo@0^8m$G`c*_tmT28vP=YpI5MDtv(D1>O@J$ug4SiCnmG|y??|tGx(8t3 z=QPLFrOHhkq`}FjT60nF@7w~Uq%fF{Qn6LBRY<~Y6@FYWa8PhKYLLAX{kJRqcmkoc zJ}8+mqWUIos+Pd1&?bc#HMJxys9R!ze zWbzB`2;;r!UButySqpcwrat$fwnsz!Fy|0q1)~KtoZ2spq%sDzL;v0Xp;Y}pJTzba zKr_wI_(_9DC}DJ28-xJeU)JHgH%#zA6UpQHKLMx(lyaf*(}; zH1G496z~4Gf53X~HWuvV>xB{M8mU>m0c5Y= zFbUg5>GC$EbojQCvZ$}@$U$Afq1XlJnbdxBjr`^4_K=YAA2^dYv4HrePdb)A4B zARdCG29>tkPVfyU!t7PQ0S-WAtcH-DS^yzQQ~_f7x&<=CzMAqRfa?}LPk{X}xZ~b& zR=x^2XeXrFc^@z|TXD2>8*j{oL=d%k=US1$5@Y1=%a4lN%3Y=}n$VoFmlS;74fv{Q zs8WJ;o|0etnLYou8Mvh}^JA@IDc3fkd{w)U1SUhM{C_PuC%1THN;( zi7Etn;s^?1EWCisF?TFzU3kf2!jJe@<0X8C`7B=U{~aYDprgkZ%az)lBipqoDx0yN zBr0gAC-eT_#K%vmAI*zV%6*fAuzTo~h zC)p0!O8Y4HNg%he9e+TI9<6r}`__0p2^D87L@X2p3>z%5(!|0qh+`)0V_& z00QfLAnK?w28$;kB$gq-;Jjq6;$ly@<-NucPyA|A)`>(}gmt;?*?n?ZlooF$T#4P&%v;B7pP8uHw6RMK@XOu{TKosP|cItL%VIun@cGtV9-INU69rW`jzja(vgaW`a(+eNPV`)%Xv+~JJ+4~AJuXH zoW~x&BS(N)mXWat8@Zbe?z%H0%|I*e57p+paet*ElW|J?tncijfj3DBrLfE-3EQDj z_kj45Md+W)h8q;d&jPmejBE<*rKgQ98p25@Oc%%U_x1^}KYgko?psgA3NJSM5Vag? zgQI`TtJ7+J0CJBhkZ!y2!}#Z5PY4NWMBvjOzaSX%-+nIgxZe=zrd6HxL-jfBvIfgI zDH!zBCEu?s{b2qm`C~uK6)@66_T%B!_gHopaCZ~Sc^({AdIIyzZZ{lH--YkNv0;Gm z9KWhAU|jJ^>%N_b!R?;Q#_3!j*Ohk$m85vABu_?N9NDvH|)9#tfKZoQuP`iLlbgl&)&nO zSNOP#t?XX^dI0y9tuGd(r184;g7{+W<*AG+L!4L>$+E-cHk{&BOy2C`qpGnLltA7) zbSw`Yoe&$wT^M1=MWA;rw?~unR5-U1rVTnjUD;9|U7gi$_Wkb?4JUJ%&e>duI-4cm z5ZSUacI>Oc9lxQ^C2M~dskb;AUWl5Kr8!pk&25nXI9`nGNu3^Y-`(@WLo3ar5Q>Di zolSB3u|Mw;Koex3@%i>L@n)b1cSALE2bzLMG&7jqY-JRJrd%L8#S)*+nfB^(D!^f& zAeOa8$9E-^l573vdm&_8AC>NkE)*KcvrveX#|XQ#0RDEoeBds4@8t>2qT{j*>X;&jWdrz1J^=@M!CelL%Y9At0d zK{Y?#Q}@rua`BJVMv-A#a9-PtpQ1?G+2XH8p1mlSv~2hp#Fuke?=-6n{ys5YQ8_iU z`63Hi19(A_S4=(OjYlH0SjX>P2QM)7kk4sq^d7JUhpx6j_azr>#T%NfK!mBKF`%c^ z>KkB&=gZuF`y4n|M{|^O7a{Smgg5MY$&V58kublPO~ra^W&D}#m1IR`(KofGvfszJGWU zD%)ChKcOOlz#6!acl1hCL7t`E>;q&{uD}~u#I=$Ub zPxXpCqF<4I*K3^87hdhO8yqm4C*11<$d~9{OABsdc1}X9QlMLJ?6Ixw%r5Q?sF6gb zBC__OVjHi%1RaoR$aag+>=3*@Dhod&V!W_dvJZdMD`;4xCB;$dID@T($(J|sECk-a zIs5&>C>h-@P#?Sc*}hNW5w<*H_`x6?J4G-l_jXC{rl;80}T8Y+h#d`XO%$Kq73Q;Wn~)b6+jS27`UDfRE;z|&$En9QJVo4Gv!CAG89 zQ>ZW-NW!%Q!zd|P8WMcJt>Rz{ZBodD*K!eV$DHt1Ecggsg_cepAh2}^){V-10-ID9 zVB#e66gfZAk@*lu&oUZ59q}7Vi2XsYn%2)j9Ih)fs>D&!c2E3^gkx(w1wK3oi~2fC zhOEMoJ$k%~t|>`px}3F=FZe^6e1S=VQ%xI&V$ca z{)9;aY-DiN#`+@t_uN>v z(>I9Ox!|2PxEbovYzJAMtppaK8X)s`Wzv4J zQ(aByy&~d2KpE-mAUQd8Rz>I)jajLDE_4D~((-c^R=8EIP9FD8lW} z=X4I+8A{I<4k^l}8$G|8f z4~~KZB;D{pZAM8=vlnX}?l&{s<4ww)5Cpp((6|=fHdtd2C_%x#yKiF=&HT0|1{cBP z$jBT1pPSS`PIs}5Mz&6tuZJ^VR5(V33z?5_C`ZPkUf!~_L<=z?ajgJy7b#<2&wI&v z?b^!z`vl{FXOEkno<5NQY>A#6mTz3h>>M%MskoGI-A~VuXTqj=bv9vH7G*x$TX2}r z8wPW`08utx3NeMX>jFlr7r`Pc8`Qhp>_zYT{@to`XJlF#&G#v_;?1)STbX`sn1U1e zB2bySqIu5uj4IlMp8&JlK|8`GVvK__fvjet5nUS5RvOfB3JIcpfjD{%CZT(wtSK^U zpd3rih6$Gz1%3FQD9hY~S;NrEahiEBtgk04_S1b%;QX^7N=xVQHjX>rs87MMB6kTK z1IEk43Pj4G*mr@9d&BOn#w<>z@2PJGr7nn?QE{Tsp*MjV@$I0}RK#U+^zTa}@b<;e zf|#;|NhDUeYq`>COnq$Tf}m%VGn=-Y0UamFpPcX^MffrHz;O1{k|4T~gi~s*2eejz zcI1oB4?$|kq>?P>FI^&L=R@n)-CU1$d;Jm->S4+N%1WC34sq`~r_@MtHqvrKMn827 z^VcYk-IL;EP7}>zs_L|BekQyByj7XU zUc|`2__c4sQLD)NLdM_p_|Q0?S}rK=6CLGKC)>GI0<7a%nVM@qZrgx1&-f1>Ga@B9 zc0Ie8I>o^7*_m*UZz17bSR8xm;xEfzTVEdTk7c_3+av+H11KGuX21FUiT@NZBOC?0 z)FsDLPMmNu-79dzxy$XbxOKTb!eK}hWj0@!FDu^u z3h{nY-X95`2$0;&y)n_#x&D1@)BUD_ZYez@Y|__&m$h8f3}ceY0^9RBVF?Suzb^@Y z4vz&e%MwloQIaTVs7qxUmKWXU*K3n>nfh)DUrlCalt}&e(U{3!0xtcG)E&`2014kq zg+z;j!#8_<49>x={9|CE#JOSF{W`S=Z zk&#J0wf^Wq=}?{btm1<(L}TxYNMT-cTzP6)s2Q}tdG7cR;dpBlhOys$;^B$;L%J! zA)H97>vEH;#O~v36#&Gi*T@840?Jr=Z$zW)JQj}20LXgjyv^vge~(vBs1!5+T6iOy z9+1O)PWse+02tE=8dzAWi!E8GhJNFX1_@nQKK|!`bXH=UH!)3CN_*$Ji0W0Oam)wMaM&H}Ky(JL2VEu}dTysuelDPC= zRe~46IQC`TlV`p;>m7{7^+~#+^!zO}`{H+j{K?K?r0miq zp0m1NPp-{lmyHn=-HUcBnvi)73qgQcexH!P6|ZF|DUc^B6&nhlF1^lQI{`1Ug>CMn z6M=g(Jni>o{~i+jNn#g$-kUIA)3PtRmYkC7H5`paQ1)8-XiJ}fA)JL*Nw}w{@P5Sa zoMVdB(->P}|Gfn2*vHV5D_4rU^~xXW1Gp$Buj!G#qR(v-nT2(~U<=@fsttD*(|>EX zUktkt`GG9T%=x?ysTqY{lrR*8Pt@i?_ffaV)0xYceL{pM&AA`1iE7Tl4EwyjL6OsJm#(V>a<(5;UX}bCKn++kE;lK%~uR{5mzU# z-u^$g0pFBG{4U;mbDvPICRoL)3O+u2Vosh*JuK5?KdgK6JM)R| zH+Xsf#2|6P$;)@Cn3cQ`;E)NKnNLor)hYZks@68HHIZ&tYj?50QSQ%Tg?zqgH#w7a zzVv3#%=D+XFp9go90J2-SI)bV1{n8niD{sczro+8me-s3-c^`MrO9vfs$>0J1P6zn zgGN=|hk?j6<2h+mW(gB5(iW5QupJA*P zxNE#&xM%w}cYs7o2K8tcOyhV%8<#4t*7I{^e8uQ6zKM6=dN12)YznnEe4=yR z=&PN}x@g4x6vvT9iAg00UWu!QFZ#IhQsn=fUU)ZRM1o#~Xj`faGc~IK4)<~@z4p-f z+~*0XN4P6GZNCABZR^RMBfxHm+$`C5d(8oyq2EqHQ?lr!w{{A!Ama^+VOTDez@nDN zNm<6$*X*0NhP3Ul@x_79=g$^`LVZ_ARnsI0;Km7FY@1Qh^J9_0$C14I)ivgwT8w~N z#5aPLuuGc~S0~T2#KGhqP8{C81iR2HY4#3B`#zfIGRl#3qW0f7vGPxHpTa~K7|E`V zs2oo31+QhJiD4Kyk<3(bmq^zINw3d$-UMweK#1F_GCwY)-JbP+&0I_Fqx(#q(UK(> zY9!nFm-32l8JtuxFT&Fy31{`f2e$6)Z#)@>lDR?i-H41Ognspk2wqu7Ot9DF}sJi009^$7c0NEw9a+6khp7K|_dJL+qZvCap=8`9Eq*JNB@kU?Z89vtp=8{l0gjT z%cys@KpKMIFS6uiL_W?T2WV9#v31AG>cB`;0z<%v|26gTjqR)Pv^7boG4H=+7*3HD z(_L(L<3XtTv~DgOC@q6?VSG9rT;4KDUuYJ(pFwHVLTXJZkd(T!@F|0;U_^Y9pRsXp z03^PWE?{heqKTC>zI34@@+T$4(pO&}+~1Beg2bv1PjtF3%}i(E4WT^;)#}6E^68I+ zyRx8-QA!t=6DTKBvyUX&OaUnqIgj>1q@M9rK7M27`ulJhL~AVoP41FCNr8Eyqj5K} zE;Ah_^OYN2W`yID_;pBKRBcIvsd>$`F$4Z5Gbx#^-r9p8QWQA>pE5V%G`iHJfI0UY zNcX4sIeS5iiKK8h=YRDI9%qa-Ba)a(GfW69Xi)qTg~n2fc}`XOQ+JwC8-ES$+&+dK zDLNQmomn0p3bS#Kzw#Scws8E3TGpMzq=;ay-POJ`b{`d6#}GnzZ?cJZ`yAebedsz! zLJxCOsJgkH&mUt$$Eu?jHnWbSd%6+$u-0K;j|1W#kVOHGmu) zaa;PqagYH{Awoa(AKAuaUX(j2Zk>6 zE1R+hM&8}eIAKtb1YnJZ`?=@|EYv&WSmnIaB0~1T9Fs?pB$dZI9>Pzk;|*qE1t8S< zaruq9vr(4x7?!=m723BpP{5}#R#H|Vdz4bPaB+26&QSAIl`m!Ldo^9?MiND&NFR7% zT~<6YV-%o@bvO`}Q?WAPlSFN~(@8+@Oz{u~R(&3bdZxW4rQsDP%R}2$up6qn;hubz z$L#NwAv~(a7v|%3Lah-l{MR+9w;%hyLRs4h5SFT^v3cM*)=Pom%21`#KC@7TS6q!= z@k_G$>)Wr`j=1TeRCizm>d%AwYj>7iU&SIohmyjY5D@1_Q$pgSvFz$A0Bj_7^Ov@? z1-sn3yWkdC>KYhLO#^*K3+NaqDevd!wcHvqc#l!h@8h|>loSjyJGs!Um(0|sd?mqW zP;Hw$q+je(9%L4w^2E9`PvZE&Y!=)z?1DthVWqw#&~YdR)0+-Z+&K6w-~8e~@i$Oe zW+}lvP*P=XbEDN}B^;+jFS{Wm{w0lEFHt*=RY;5Km$Hc6)-Mu_q;<}KOnvr13q+lmFv@XQ2iKfsD!WwD=L%V z-U$#+?>`7)$xbg){S$)n2$Jn*`(?ZHEyFy1^JI#FKa#^Tk{>R9mg>ns8UFYR{3g!1 z2u(d{O(VfgXlPi$XrKLoq9VJ#8F?GZqH!k&5CA>?)$69<;B4}d)DtYTa9l`SuthUC zkaWJ&Q=I?+>|tWC>`nyQdH|1OGB=1aXB0T(BJoxrS3HxNX;8SDhagks2mB;EV2+F? zRe1k*XgS=_z@#8F1QjS*{0OWn_dS}d0UN!GwS%FtwA9t10+dY}EIhBd%42ou%Sbuiypop-8E0n@|PR}bzvzpcwrIA-cG}^pUTkZ#jK7L{MOgC0bZ4)MyWp z$ake{j($-V-5OI$vj@9|IjzsQ+RGj3YE3bxqF(8*L{go@ZK=fbmGJ}IRX-0gXh{tz z6>^NPprGWTFnkXfdGy{K=L5nV=lY>oMpD!_7P7d1)9!s(eMK=UWhRM5{S+i^s))1r z^diU|1Wa?!GtPy7(|E@G4q`>EGdcd@-?#kf{la%Uhj=v)s75H96Lt#S*C2yy{cB|u zU}Y?E+i7SwK zj47^Y8%*S(|dh^H$eJnEX`Lg{;53XZDT7R0oErWulfFg#t9%AD(k*oJ>Rdq{c z=o;T3H#{qBCA9wu3Le*J}oILZ&;$!8du|^g=@aM zMEaLuiP8i3Xm(#gE&s2UJdDDgcy?Lc80meecy4-qmpGHjO%>jBXusyn_h-j(BT2_E zDCQuD`tXa|b4O5w|H6bD@rd(!OXI=n7+d*!FE0d`FIVfW!4)RR8{Qv;T30KgAitRL zA1-z#92dICT?5Si%j6H14`_-{0!4Z%J>{GDLMGiD&NIgqHe-U`w`n5sMV13ozIY;$ z{!o|)O?T3!>%7tjabddty9bsPjS98T`I{u)q%**@V}ZNT=EN)=U^Ewh{S}OhgMWQI z>ty`%Fou_v$Fg4lfzooztW5?3Wllw4FU0nB;@02xK4gprQev_9m=qV)S-u`xW4t#M zNz69s^m1ys?@C=SX(~t!UuY622>;rt!`ZmsdY*HxPsY5cm1&%&%--2Zc_T&4U^2>t z8XID|s9n+JOZjW#eSss1Z`;PAz)DPyvJCYs-Pu}M+j~-8Vtgba0&@B%0!&FG*c-PV z_XoI`iS?cy^%EdF;Qa_IMm#cG~02Y#s7UFk1#$_g}1zF*W^kW6VolU24hUzG4|^%c`^1mw`<0TGjQC8r^zA!SnZ(*#I!|~Trf7L9oa4MK zIJSAf8^eec^hUgLox{uV^tY}8Y=CDj61UfP-93;D*)Y6u6{ErJO|}vXDZaY8V6jz4 zj>r9?qaJ02M|5$Djx;i>E_OQ6|0)s*>v8v6G{%W~R_1vz?GJmFkT+!&oDlBHo4$Qi zjUd5d7fS)&GIT~R$1VNJSu06C61dC{R?J3RQ?lV^AAdIc@JvfK5Sv7YCaQ7i30tv_ zhrT2eybV^{x*16)LmnB?NETBoM7+pm9G&Vh!>$&=%n)G5gjtlJ5Acg)el8}!q!QvQ zc|^B7<#S+EgnaQ>vf1%X#+)!oapDtfYFBit*#e$MM0a0N;p!$Wx0biNs_Ss!4F$`Z z>K7&{P3bT-rhvF=@-XYyU#%C9VTpaJ7Dj3;>O<(2cwWvE;D!TBD1y+=DD0ne+@LHL zt6T1OWS%b-Br}#vfgV<`mFiq=(qCTYg{xQ2gD0JEH)`2Rip2^*v%-vGrRCDvcvG~H z%Ik!RDJy^9%m%*iK<^`~swUBSJ`j~^d!;INriB+ zQ3+TlowRG_<)us%|ECxvmVki&8R-~ZWm(_q2UqF0_nZ`Z-z6DC@b3jAfi;NN;Esho zs?MGNHdSh&T9e9NRh>z81=A~2ay)uNA5kFo<1j%r>#A35{WK7{lSrMUg*HGT=zmr# zxNoIZ?nLTKapz>6&$};{23@WQ)9Z07Xf0&-V|t}J%dT@9QA%|ZfwQ_98fGjEP0Ul- zu2H{_gqK&pqWt5OmxrBgyI$V)fuFta@SkObHNX|W390LkAW3=gh)Dd3q^N>h0TDkA zlmDm2vcJ+-4wZM&KjrU6|8Y?jIY!)~c*~rd)?#+&e5t&@wpRUl=KrhKg zV&*OIB+h}#clUDJ?e{95CQK*)`?Kjz;eaRid-6loDOT;A#^0X*fIl5yfFH21VteBm zvu10EtlSs3al_Mb6uolaiC}V#mzWPBy1qzgChzvjxXUz+mse)33}noI%T7|QZij9U zYyFWJ7WAuU&>9OX<@=zr*&9o@07Dddfqplloqst|WI)(A6l83l|Fi~Vb#^^`(YRWK zgaS?)1>DM>(!-hks5$(ci&XvS@BSWIJ5ub}+=m-|SJEg3A0YY=cNsWP+%l zj@UNFQNj9%RiYnn^-_>%cVCS$dm}<9^zw`l-Fjv6Qyr2Sp|*-{LRijT@+`}PaiwDm zltbO;ycCUOE&gfpVq2lnhi8b*nH4GiuKPYr49Ukp8{CL$?)q`|mw+g%3ud!#TU>eL zvUJBOov#GXl(hB`0oCHOi+1^+VUM1Y7t8Ee))t?qCtcY}Q%yKw5N`>QzR+{WV;bZU zB)(H+Y78lmg{83ZMiIa=RpE;hCt`L_qpUHebn*>=q63x(5TJSjvIk_Tgf5=nzXa!~0}8(LK+ItZe;&TWtRcHhHhFe1vu9e>p& zyxQ&bwk90qqhrr9Aa>F1ZnNFT(IkE>q34Uep z2K0#bvTA;rdvL4Csf4aw;U^^MH)_N}@=s;INXdl;blJw8fbeP+t}kOz{^;-05#iJi zJg@w;9M4MsVrToB?vMo}G1-v%2->{xl_55A-G3GMWyH4vnu*XZ(gy1#Qi^DAx!rFS zQ^MVZfe+cBsOkZpfCbpqwMc~3%DTqL?*5|PA16r914DkOE*SV{;A(5k0CG5M9>FsS zt<07Np8UV09r*P2T}y_$npx4yB3aQtVab&#SQ?&%X;VhBsZ?vyLiLm>m&BxyB2Sh> z9cE8UK#uh;W~HMrT_3#qc{M6{T_ik^KY$G?h;Yk;EjR9k|CA;FVn7t|Yn4hEA?P@5 zenYznIFO2nEY%58KU9u_{@(>C%A#d}Da`^d);bYm{RdbH3qnpO=CX8(VH|shkGycb%E1B9F|T` zw0qak%|rX4QK;}CT+8q*yetTcW|h7G#{T9qH|1f@R?j)md80Y24_j-k_cnV9pEodu z1#Vpho~a`DwXvgxGQl*Rzz|e@U){TR=FyuAIgLRzCB_?pIJo=W*l&eBn}X_D%*1Z^ zh|+3*ZZ4BIsws)t*Hm5JLBwXiF_8+9B}5y$yp=Q`3Ww*pXX-Z%8`!1lFdQmew*7{| zwW-qur_>}D9&0D!sspmYbQF$4fL+GpQz)E9u<(HBA8g?^;_eng{16O(Ma#!!LADSB{YW`Wl=xrY^xl6W{mR_S~4qK7XL=mNJhUKy)6!6Uo5A z_9spUU#VhbwuV@LZTd|FD0xEW{VPwz?k_ReCEFPKATHG#;EcVGwR~Xwj(+!qqrX#6 zg+a8!yvIz}=HMRB-sNhA8wmd1n0v3MHp+`LUN$a;?WvD)SB8$6na|5nFb;Yi7zWtX zn09pr!ITHEpE#gFK*B1UG@OhjfX2IH+eA(=Y2WmFV?uvvF^uHa5dKbL)NuXI8})54 z{>cVf*H&xL?P2{4=_dteH7{R-E-i!9@AQHOy#goTckaFd`Xn7fDC=#H9c5QfH->kg zW7p;g;}T!sA0YotJ@yjYBs?x?n?k}O=g-uYgN%q)Kq(3xEOOCs1D3roeT6;<1YC+`8q?jf+Mj0a8_-UsqgK6YuVL)<+&3 zpZ3`iTHi^wb+0t^Q)_h5>3|9?|t z<7x6Hfak?KFl%}RJcJvtz?$>gi-)Ju6g%T1uCqt0W07YYVEg8veJV6;B9ET3ds%*! zcFRchRXM4wB5UBSbMSgZ^2ahK^^y5=dLqd7W^Vt?Og^$(%D-2CmzuLjJTyjbPX z__NE~$>T@HK|$rj)r&6KtfgUtzmHjl&A(@l3ZlpM9ctHmd~r$8bTPZ?6o42EosReZ zgJHq;#FZR|D@!RNf@=n+mO9qwo0P1`05Oh8p*iU1uZHoY00^R##FP_?l^?&S0niv3 zmv%REr|Z7 z9BL$8SVp_e50 zpy?w?nL*n}MVTZ@6=3)0>fKJG`ArU*Xgs6u&YaM5xZ5sy{z8;&JzhA|*&`DU)Q^Zi zIk$y!L|3t0emwS0<^(sj18yAQsms1&2j#chzjYrDxKCPM%*IE67v!ev^X#7YM`Jx% zdnTfgsREG>4mC{ywLzJJ70l@jJ%&@y+tQT;+f<%NrDb=pTfLJvBo2v`Cy8`9@6H}Q zEZdLs>-2a_f(GMMXcmnfLQa73!zAVk8pIipUKF9>VSq`3&_~%PGUsnW7E=Yox8mMG z_R7)&VijW)gV3~ZrN^pJrKn2R{9g$LluOLRk{7%MjFg`k%V1t)`^!iq$Z|*!FF3aa zlN^%6Cyj71lg#84P`{$*Cu^fptmwk6cqEmWZ|wrN>D4JI6)7hFBPSJRlDCN|T)syX zZIY6!%bBMv>Wgpai_?UL)HI9!eiU<5=bTnN`>ROPAn7i)NnQc#$*t~BmM3Sok3+4n zWO8!aj`vqL61`eBHVhhOP@MwaY<+;7O!~?}*W95q+!RRkS3Z3y|Mk(ksn-Ae<6;Da zsf+KLhIjve!k{XsK7nEsnQsJf-oZnP2YEZcP6FeYL(gx(V3$4()E{-(q4Y;9lBY=L5lJ zG#aB=xbf*%hX);XCO&V#{e^C|L!&qyl$W5D00rO7ToAxZj@=3<^)hd-4fwJCn70#L z{c33L|10|Wp9lY+Zx5`}+gZQ90hx0K)DA84Kac?t^ksXd6d3Qu0tfCIg1K8nDI~j(_UH8dT?Il8XwT?Nh1vYdhA$yE zM{BO$sm=^DAnMr7$!#yEiW+t&9Qpg*?1=n6;#FjtW|1u z`1Kh^brp}ein3<3c1)H^=HnJQS-AC`)eT*-Od6&YEu3sD7*M$YnO-S)fmURJ26apyeVI76=_HEsJ)fyi=V`qVLt z_#&Av(JDAyeoLmbB3#roNeeroM~ZOI<3?|kE92c?zu!7K9r-}-XLnIx=lR@tD=My4 zm!TnAqtBgP9BRv3pLO3x%P5>lqbQJg+k5O#;`!%q=~U{zPaFx(;PZ@Y`&lW%N3a@{ z{>kR2+eo2E*3w$Ja?R*ZzQpxX1`{*YlEgf#Z`v@(U??d)GYex8FSe7=6H}3u7%y8!C>6 ztEE7vis|!-$DDsQ2k<6A`&mfBa?+-js1po;U{o3;sE8n`8Qna$#;W-ECh`nB2@v~_ zEYB^DvH|1Yzz&=)TSYPMB9IE2A`B4->YYF|FhDf|ITiDrkAd}2HQxZD4bbUD?9}O* z8&%@<1rpnGT0-L;-^M!jQJm;YykwHk>lQ z?!{b-0(qtNg%?`nK05g0hfV}KRJ?DRq7I#CZuma(Fk?ELmR@M0K`C}YJI^#xtzw(D zmR&M|V^P2@!Pw#cjjtPiSryz(lY)7lxN|Q{W8;TeEijW9I?<M=X9sVf`h+h5s#jvAX3 zS4=sidu-1;6jvIzJ$pmh+Epg9o**|pUw1X_g6GWAg-7I!(+)EeV}0-rMve zfi^F@1!LZ~Z|T9Pp@Mt&ucApxxs12n_oCn!$$m}yP4LQ@MdUM>qEd|{=qo-Ok)>~i zWeSG3n4X6`0fu47%Z5s)sg0?YOQ^mEc1~Z=YZ-bUjf$T1}+CwUr}oy}WnL&Zt9XrBY8|#ib%-#U(LKdE_kDq~X)i z6KnZ{esBIRy|ej>Dpn~Cu2 z^If{&sa{8E6EqVYpW_pLGIwm@c`qaznY9e?hq6pO^IT5M^Ilv<1A>g12s=gN|6 z>eMW(Rj<|CFb3CHM6ON)ZYN4x?ZDnNBOv4uY8FJj6vE4p6`!c8b)sQ1q>{)^End&+ z&IEoiQ46h7VtVJ30ba&FDH@rUYbzs!nL9`lij`yUgI(^`TRd09 zgb(`iO8B>CxN#^4CuLYAkGMM<+&1d zYOHdZOE>wlsO{}8F}hVO)4pgzZHx`&@euw-$VOr~trR4==zk&w`Dv~E`mKiJ`!OX* z7EH154b|p!nz+c!=m${5dLaq1Ew6>$+#3)Ib&?sH7JQkj7sx67)WMey@BKiyJoAvS z9LVBa=W#>98q+v8is{x>u+TXpnR3z|{#(f`j_+63eGZ_Cxzio>_d1142lP3>EW0)5 z@eF)uT9{)KR_fLK6ISxU15)$dqt58R3C}+7O=dLp6l~H~&UFN$&9*ztGQL%9>nn`{N^g4hE^G$gn<8vLGcVwAcr;GaaVdA&@ zXGhY%w>*zGaDckK_8*AD3jrH}0Zj?8BPp6ef*bYh9uwLZQh!;>wX`UU>O&Gsm|^^4 z)xHCR7qw{aBUJRWOua#b5!5>#ewwuKtGuZRKORVh+p0J=96Wf&l2S_!tAU)O5@}%p zt}_67K!b|siGTPS21;T=eTX{+8EC1!4NGys(q-~+Xy=&#v1jMQJ|KXgVONcsd?D7_ z>fyiBcl7v-7~2o1ff>Hh1@>iqV-4Z)`_^CF{G@fKlk&`oVB5d%9QgOyz&78t6Iry( z1)G>M)Jm_=dHcctOe%T}phef@Mh~EfQy*E^QuT1u#XCusJ2RIA&F=;!e(;jQ2C!@e zUdzS{IkhNV@BJr5qdtTrWEXC=qs9c-cHcu%h96)AR(e=I_X*+!Uy|Me>*~0hlc+m& zpVp(h#9An2|KY!{l5PPytB*P9kq&8Ytp(D-X<(4WqKzQ((jg-$`vl|}WrJQ-30 z0pcx?Ls2_Jf@H|PC8~-@wIVi1jCf?Tz=hA=+E^8$pz9IrYmPY+A5aHJuNw-MnRKQO zSTi9VlRcs}N;woT#Q!w~^&vxL;;1Y@lMpzyl-YlTf-BDBKX4uB2Fi=R?Q3Kav@5OE zmH8{c_;#=Z`P2ggc#cvTcpn+i8S(m;XGx5K4AdctCmK9K5KSH|TSVm4-}8NmfuHVN zd7+sA{d5tc5S&Sv)|C&LaY?kMZAp~c&(?*JAKlPlpHJ^EpA;wi+ZYw2+rdrODrnSH zJOLcTJl~YV3UEH;0Xn7(0@qPpU|JYG+W?|e0lbs0>!EuaKcpJVq~2Lk1H!?9E%8JV z_p6T&Dg5QmHCz@WFVb@vCli88c@|{Va&zx;OEIh<*RSH~f%fFvUSbuxhHEr@~JH&zs$*-sx3Ny+Cu3Mr?NvMpI zy@Bs5nmp8cjOxzG77?Lfqjw8?8>9A&_xI7dJ{Zo(y{ZTTF`8VC%wmRw)TaL32=FTy z=!@Ni(tM5vVRPqdzwz#CB&0-KRQeO_=uf2_@`es%FH$Ti+U`K6kPoh+f;E0-Mee{{ zPPZH=ErjtNtkR}nIriV|tG(y~pIz^XA%3Lrcp34{ZnYz$9XO13o#B?{-=#kZ4=qoA z(&@qTcCnO{Tk{MO_@yBcqocw3*7GO2XoajbkS@st zTGa*6k1joY1;C337g``QvS5^cm<@17FT~b#xF%HkRQAwk_)K6JjGT|)Y&imLp$~GgwODV%+#6iI zOp?7oC$VemgiCrTKl$GaW*b)#90PE}?e$m#<-l6VT0aFSSVDnvsOCm`(rt=r{5=q* zo`1il3`j>R09j@2y3Gqu`@69T3KU`u@b`{)=+t~MAq`^#l>7a7mWr8RLAki`!cpG> z+Jh zF1rt8c8cWrxP<=9uzEQu4H{^XiT$e}qJGl`=q1=Q9}ir?Lt=ud&bI`Fs={_W)(heK zuB`Cia1K!0rJJCvU}=&<*YGTK@X>r>$kpFp=u}F)2EPVP8gE?G_x!b zVmiMXkXXd=m_Q#U(-PR|N=cs)!U(fbsxLR}UHT!zSR~Cs0Q=n^;lm2I*7u$hPBSkP z8U(sZJ(;+(X@oCh_)-;u@2L*a{pmpZl9j_i6skN8N*yHYM?_itbC&(z(izZ=i-JZt zkKA0ZEirhI-3H}YgB(TgUgU>|qh>+&Gu^=ORxIm|a@)u_ee?@dK9}bP?1Q_&F2%_5 zF33L$9cQ%=Ia8+}C=zx%j-ORzG=UpABl}LQei>$Bb;Er+Abb<*<2e2gQKz-KJ#{_S zc#LGv@8ug`0=(v+Jw1Xgo6aml_VJ2DHZA#&QRY!TN`T(303ixp_oeB;3ox$}*uC#W+L`p8%*EFio2U~AZG!HuVo>kD@# z;|5$)vmju}0e_2LO{4foL(dccd|NH6H*h0k0XnvwXHk&LeI~Yd#nOGpMfSfXrk@m8 zW?fnT`8#6p3-OQRgLN4?dK@5cFyOBvtmlQ8Bq%MtTg?P03`4S>kI29X>nj6dn0dpO zcbyeR%7~4htNj`Fi4ige7le!3?*VBJutJ7nERhH^Z${xy$@b3*!^8V9E&!{$2d;v2 zFeC0Me57NYH{Seb;r3w;iX#u`oa?$S?*K%<19?3+46tfpLLICIF199=_k|F7zoK=! z>FK#3ZbbG9r2FdS&Nk4+0zsSEk^5ti;(|*zl%0d!aD{1QQ}i0vWq&%l4I77 zYE$E!H^f!ylqj?uFtxp|`SJ=q2Y}gqr`n-(Ez&NjUKU>&xgBZbK%MbnBuXklVN?=t z#F28mgIjx|u0cuDu~+go*=$@7C5P4SWc@yfd43ZKBWjQ*%prykp61V!WMi9~=DdozxE_52<=&>NMSzo~<;b z%~*d;?0EeB>b%Q4eG!lJxP?V`J|R!X-j|-~y@vcs(kpF^j7_yHxy@O^S}vSTxo4uR zNGkuUs`2}(R-bf6% z=|I8Ydb4R;2A2IZ%AV`j{@-0BgB)L4o}USH>HmT6`?xX0^*dT ze(bqUL7s`z;e&wr`S&JBpg7eaAvSck_u#Ym8o>CnzkYbq^Q6FRw+94tK&=la_|t8{ zVC3C6Dl9yI*3ga_#nTa`&7{nxqob+Gg+t&G@Iy9`}794XI~ zLaZN!|E#snvf;67m*|T9ypY&s1L*4joJKD6B z^M1iak7E0wpPtgXZ9yRZ{lrk57QB4rl2eO$>dk_l&CkH$R= zSL&pCrhe!tI@YRU_@J%P zE&l!T?F~nd^n0a;<)|)Ax4BCj+;yjzcK!LLM1G}6&rsgIqE+VgH7ZY9qB)A5&csWr z%zi!^0v4O~jw8How4NR(?=M}T0Ut`p)!yNrwjToGiy4?hMxWLx{fsh_)HvG*4mjEm z^Gc)>F%oIS#vsx_8JERH1+_!zh(z~0i=#8cKno@t^4kG0%j-#}`h>w~0x>G$AYIxn z$Lj!C20K?t2h+?t2IxXh$#TMYtdvO=H5Ifu!H^6|5*$Hpa04WR46(L|#6vfUQ-69K zREc2AZGn2&ccFBRm+L*#Do~?@;~V2@okejwfN--(ZdHQM8Mj3Ai`0w8ttR=zRc@9o zJY~JaHXGLs{P(LXA}gccx+fx~zq-dLZ(Xe3s&*+@s;g8sIH8$Ekr&GCQR8kOwB_Pn z@*XF8gEulQ7rRN_!n*e+rjQuZ4u7yV%}_L%)TPW1g? z9YZa5X$v=(Vhd>(ojPlrRf35c*yuBWcC;#q*eXqtXp+06yGTYEI?hbz%5S;lnNnm7 zM*G!WiQ!&#aiT6gO6gw%{3m?RXj&Vr~2G6IS?eAJW*2%Mt0_e7JoVwfkAo1|B{LErfY_(=+emU#4ez&Pz6 zC`|%ywNc4&;wzvuX2$}Y3X>BYxT9$B(~bVnMbYkp%R+&xq}F2L<9i(p*7hH1iC7#7 z(pL*Z&)G6cS$HJ^Y3nA>=mIr356cIGwpMSdbM-JuBFNH}iVyA8u$U?+qGkc`c#S%S zNKMrg&0U1d-Q6kF;S!07Zt493{?l09+To=U)lWa_bgc<{M6@%@gGW3nIknOpkBq>G z^pdA|5 z*2-1-?HTFw>fCF;nwj8PX8GwI_$bGlHW^j?IqmzvebE}+#Cw)++SNttxkuj9J?kfC z4;kt+yQjN-rmKV@e#Zs`*pMffBh;m_lCrtjXZKYAL@Gc`O3&;?+}KdOq7<*UyVPAt z!jD`dsLzOKz$NqbAOh|oINwj$_TbV#iLn7)Hn3)lh^|>4wjVkVh^`rh`TNPCH=cZ) z!Te${q008Wut_`9G&n!T+qcWb6h8m!Eh$DgL{o{<|aobs<#YJ!r%_s2y4iwvZ+hQkv)SpsBYJxcdT(PQ>V}aO;V~ z91k_o7HUtR$rLZ<75=$j#cWA_B3D}2_YPb*ps6!K>?hQfE1~!>bv2<1rmXg};9Dr& zKBLv>NxkoHamB~X1A}N*sZ5|k{Exz0QAH(&0G2j)f50x6j5xIFG8o&Vk#Hn*NEHWO z-J_0PQwPiSLC8dGquEReaajx+mB$>R2uTOx;E$>U$7&4}5B=M8qs9OgGe3w%xEIaT zUH|a#c}i`GWAoVwEZDb-tSTD6^*JawuXyaA<&R)MneUT1T2}Dq23?litAf1lf4yz! zxNz#R?jTaNfSEk%h31+_Z##p0ZlL-1?%n2NEJ~n#7#08}LQb+sPt?M9Rs6%frGLsW zQciEk{d&=CXrLq(AU97YEmko)O5A=(ShFu715YNHP<$H!rKnvs7M287fUgLAJ6M@v{s0p2ru#6DV>C(sskEDMI3co73OECH zg5Z8$s%Y>3(FLR5oxr{i6)LspYs!@5dSeLIJ-~ISSz8(i) zXXt4pwKyFv4sxH*17%sP$n>8Y41J=3^e_}o*~lLcP2hn_wmJQj17Up_G*LN;+WswX zD0c<$N+N3pWuWqiYv?i1ATis!%Ha?C6L?c$&QF{XS*tB6(5J9Af6ONj97o7C5zqv5 zAn%_sq-XxwP5ufP(Vfk_cqqJ(wxKwVHe8+))}DCUmHC7D%0cAYvno~Sx?);%$q0@V za?9Y*Da}WNwpqe^5W&(+1PBFE(rUg<7vroo4MenJ>uo->YC}+dy1eAu#Mn0}kX$(8 z5$fVO^J$?NlAnqx(#@Rqhl?fzI$zeQ1%?-QHda6KYP>FAQkhqUHkD$Dt%YWEt$#tl28&n&j(>5sMqzKmTmpdEVOC8LnM641%@I(K0` z*EBY91v<7ZE0fg^84m}lQ*eMhPg%XlHQU5v z!Sfr3gmU#keb zP5IqH9>89{p_bJZ2VgQ(UW_vp9+_{n)-lkn+b-ke%d-19uL$fHcbDQI?rTEs9%L(9 zqt|rwPI9p8K~rKVf61a|>vPL0LDzIynl(=pk?*Q#*Qc_J(Lg5nP6*GxW^-D!nzc;{By0#mJ3a0 zRa1xi*j5ZDEg0Y9MMIUdowjOmwy<^Mtec4Db(?WR|e+8Z0y?a2N7n zWcwIUvhU6><;8Ah6_Se0KTr&=BIkhhZw5emLPFqM`jA9y0(k9!t~qiGYVFq6Pqtip zM;>*fn6iypY0TPF0zf{>x!Vv6*VFne=!z_%510e@byEv&ZJBR3Nl0WGGMFw2JPn^Pqg>_xPxdD#o&@s_ z=SIrrKHw5UKL()b^A=$)W1%u(hH~i#{wWKCaq7jyA-vCkNYp9z{nEFy_^}MA>0Ix} z_0sUix3-!RJyh0Z!1VJ@pfveBIWqBE|5D40lo{P1Cc6cJQj-+jQ9P^I7wUv$p+W74M3DWiW_ zsj?avXj-G#Hax<1E9>T_A~m=vX$Pjn%AsM2Ai??pb!`vAYoNIzS0qi2(m!-flI@Ra z*?|V^Wy^P+nL4`&9kW8@gN}t$qAei!Lj0&a&#h^_Q>~bN|12LRNq41tSsy3DoNIFL zypMY+$B7~C`2a?e7XQhm$DFzG&(rwOS%b|{?@UkL<-wCUg!Kf_<12Y8GlY-(5WHT| zkc)8FMYuXzvQLujeaEIJ2HNtuz-_QYHlt7a3?BW7&u}PmjDAS)Y3J0XX!}mN3+*%Yk$TVcD-EaKpG3>O3_=G@>4UkGb z);u%yPbNoq6QK7j0KZzd7w(IS9l*$mi*ezkUXHy36JE@(rr($Pecz$K19CHH(Dy;% zU;(?@DNgP)N#tWtps&dwrCStWq zVBi7Q+v}LdU*s#6A_Zm3Md4 zes9G0MATtub@K^dWEp$pC8-OV#v*t^VW$Ia|8`s^cik=$+!p0xd2UIxXGI=-acIAL z6dDq4;SW({f>qw{sllhE~_ z0wM*A2EHRi+X&Ef*?^VY`Y~1>Xb9XLkGwtu)$HZ%ce%U$$zMOrTbl%p;}!m1OX3LV z^0+JKAz2GlIc%)g*{`79De6ENv^|9?hWBLb+uv{Nm28#h+0K?3sH4u35!IT12)?-J zkFY$HAU_Uj5ZnE97>EhlX#GUD#Q;RMREIZPE=-w>5`7$^pTMY(r!xU|5oE-nR|qu0 zfZH3^ybD=V8TE+pUaw03gV+GX5iSPM?+h?U3Srw%Oqd8>Cq_uVFGWF8)LF17yNf95 zz+mQldCFS%_v>sY0)n9*GJ*iz@2=XebY;BD2POwt0qSxV_cvuG|h5 z{wzmiFeE&@Xuswl7vNFy-`Sjj4iXkk9Qp6kx1db4;V>i!{91Ru&JWIO{ z>?&vn_bvY-ue-6Lao>otGT_3@2}Rl6%2sa@c=xa5hQX%s+<>S&et=)$=S_U3zBE`X z)1A!;(cMd;@OjQZszmRLbk>O2hEPNy7}>FO)BZn2X7tV{O((u<-6$|OId<`9k{s`@ z#|=82bmw+69~g1sG-rxt4|d}}@k#E7O@{hLcj1s+wO--eExq-Li(x6vp>(udbj4u_ zl-msrYvN*a88mlic<(|;PKRlGYc}#N`@v&|2UdDDCd%RNKiz|U*jxq#bZ*qpZX?Zk zQ`)H&!g|E4SNwh@Mh64LfdZHxhB#LlsFO1*EwmTTbLn{IcXFI)hOd%pQgJmub|7G5 zV9&jCy_rE&T%}o4(vYnA$OA7*sb+t{EPXK*a*`w;P2P!pUyCt!R8^asO1HP7;(AIi zE6-8g`6|?EXSJ$2m6lazl&0i95hoYk4{Zx--q?-h$?(+t+&p^g?pya=B#3VvVUiHT z!P!rLg%W4j&q+Vu`@!%@Ty8mHdOEDH*d;o#V%d6GU^PJEnx}M0`)U^<}_4 zMI!ta*OUBQSYIzfLd@^aIf-+VsnC=3ed)CN4YfM2vk#;%iTS2T->WR;Lh~=mz|yH0 zS;&5XK4J&IK#(;0cJyV83=6OmN`ubz)?3wri)9kg-K#y(A}~`&m_8tgIpEhAgj60}FwToX#bj&m0(EF~4phn=iGl z6^~9({Qi~jeO`nwk7ALH8w^96D+Ht0spBEiZN@nMA)k#uUhnb?5CwtaL>bKBDla|s zqV3m@gs;NjY@R+6=n7*mi=p=C_a0QeL-8Y+^Fjz!GNJSCvcV_gV*##LUWFNnKU?O2 zbTa3I$My2zqu0@}NU9f6-sPk$MPukHlj7*d_VQ=ntNoHV7eRSB2jrDjUkNg{UyHf> zGht=vC@}Y_ZT*8E0q;_1oc+Z``Vg4V7Pcsng7KrX2LJ35NYjSyn{%X;H}3sVLX02d zN0^&h1lO-Kx13+s)doUuWGReJmL(92ZsoT4?Y_4pT*hobv`Z28w@P?@1PuS~kj!P2 zvcOB>r?-YXsT3K`&4T%OCiDS2&Ylf=E=}Y$9O#W#%k=xT?0swf{VK3}8-aPNQ(&E2 zZeoB*~xmaYXVaaHZ|xIo{L=`KFx3g0H02 zDqNo>a2H2(dYR(xhh2if4tnmmpLbrkdH(i$e5X!LqY~&g^3->jXyFW#Z$eVJ_~BN} zdB_#>pI8#leiJk$E)tI?#022rgO!ClB%YQ4TsY?9DEZ0;ko|{-*3ERQy26X*NS~O> z{IiSEe!NX*4HSVAAZ7f_wg|Fu*U57wB&2`kI%k;tCTYUET_GRy7r4jA- zFanrOqpQej$dp?5-osZQN}hu)WDmfoRmk)tgG7=l)>)|)tksIfqi(QED!!sSCnj92 zrU%fo3n2YnruYv=2S0-TFmRS`)Y{9EUg;{CxGp91DCH3<%KAoCn7Ft=)sRd&*ND?u)_! zcd(~@K+?TtS>pKQ()XW`ag+d3l;n6QBNQsgg>HW6knJdn04i+>QP&oaGutZ`h~fjV z8Z%#GjA$UoZ7qR;AfOFIv%Irx>iE8ZRs8)M^5{v4_K%WjP#Dm>DN0g?aI% z0f#3*52kXRoa_4?Ceg+y#Yu-@?{hG`&<09)<)V=jrUVl0z?vJV4InWy{(Qppn$$2* z;q1-rx1n>{*z*p0dor;}f<&g}ik+ewH)2uOc@BXY?w@&?v1ktLD*XAMoj5^Vki#lA zIg^ailtVi0k!J!S?ZcQ_V?aW(e95&bqtgst9%cZyIlWHLeE^ou*H23%@m{Cg)nu<8 zU4kZpdQbgXYWk3eVp(i6TeWpG3e96>TQH;E7AO|-&DWp(+2h0gzlDnmu;|w>Rn6~f>DDQC0mo|+W(~XDhTJ+7BF6w9!KJA(7)g635m|OQ!3Y{7k?YlH9f}pEZf7uf$(%cv z7Jb7~(CMT2;SP5E6Ha9GXqHnM&=`71TwWV%oRFv44Ht(gEg-Zk(`YP_W<8k!`^5E< zHP7v(#OB8fVpJY?xNk!+v;>*$hln;fNu*fQm4VENjbx7_-lpO=fl_4&)W)m}RvjI- zp6n*n3`}b61E$sP@7zV?PY{JYS-5<)>(~94#6aW>NzTdIA-EVyfSsMoQ1_Aa!sT=` z?LtvnKeaF%@Ane>IJ5^R1SluDJ=X$nnCHAB7G}|L`~EJISoqCZ^7jM|IuU%pG`{rJ z7A$2S_5pKl01b}-6~Gtd#=i;ytJsfRt9~Z~2qc*>B0gsE!YTay+V@6raUr)c(}rZ9 zk8Bl~ofnusl}se`>A?9X^UF&}Zl3)dKxk}>Gpsq9^*ZBg6cAdSAP3+t*?x6NB1r?g zz*Y?Ol2W*uv%AyS^`7FCLq=~2k!sm27{SA{wC-6Q56~r*2|>-H*!O+qLN(q1Ofnyq zet3LjL1F`9_eS~iD6K}4eku91Al6a1FqW{w{_i8HbF-qrXy5rNzdZd%d%`OJ2P+@p znrO4X@H{~k=7BP)jaitv#=k#7CP5tg{P!h_?mb~FjoX5_6YK{UDg9!i*g-PWWae{` zEN^~1xi`#AgbDJlLY{rXoW2Grp#Al z`q)e&gc3xPcVXD_gI8Zs+63y* zoehonuL^MfZy>-HT!n9lLiepVA0aApgeSj2%HNbPar`*xxbG31gNB0S%}>$f#YYPk zVgJeOYi|SLZ2$Qox6v2MiE}k^2sLJQ@C_l1wJ6k-b1*Bo3VsEA*b2DEoo&5t@q&ZO zJpW{Gyz6Ehnfu)l_w!`%+^anPPjoslqP{x0T{Q14aD71ZO=%oB_VcESi* z#ftYLCg~G6Swy}Zw~oFY0*CppxFIl3`RVEpzSfINXV0B2tt2#V$GM504FUM>!E%=# z5evV)|Y>&36a&8)0JU*M{q3 zyq@J?e17tl7js9zJ~Pu!e4HW5Ncmz8SoW5(G!@7KuO#A#{d{l6uO!DLkwlG^lx06G z-0^{KA3g4!HMr5@TP|r5h3z9bh6hS!GM#s4AHG`F-?cR0uKs#}FJb z3>~$v->usRaFhk%4SRH2iizN#SHl^7|DRWF320r4fz4RI1<;L1?V+b73%V-l!{3ds zl2+TO8!!hmxWa~W3gpa{jlW}l=7o}e)G@mc{Lzf%$5$PagU&71y~n5e0S%>60CNq? zg~UIG!s9$CfGVWqfcZkcLKV3 z%YB0e!u2mc!=F6J_d`O}t%6Prfz#1qqBnZyyR?7kCg(T{9D$hC~0%F{kV-9Wt| zUc&N(Va?(zg5Q$v^h|$Lzg_v!bkA2d)&027EBnZJ&N`^z;CQqAw~yvST3A(P}nOAlNt6A?dLuO1&ZY}HN9lVOdqaC z%y7E;M@|dVAC-k7m7WJoAnp_Q`v!TrF>QlwUo#?zg9psVC`*6Hv33oaPW=L@B<;1O-47|CYOi*|! zImdT2P|w->Ao;>$8`|k5p`7F8qwWTI4}*BZ&t^P^H=J%&k}$jO@rZ8X7IRSaO8{d! zT}0D8zcWs+GCmSuNJwvm0E5Lr6SPs?wOwWpomcTkN`7L74Es#TB-)_>Q2QA7>QwMz zw7={pE(Y&WW}*fu0l5>0>B}Eh`A$^9gqkz2G^C#M&fha8z}@^*ukfXBzI;>|dtdr! z$stLNTMtJHs#aK;AlbVOSl4TEK4=RdGi%^#_XStjA5P~Y^zoHRi_Tx5?opA`Vhq{@0$#!nB~k!4VR4<(E8>dq z8e=B0KuNwV+ml%&mh8mth4~dhTK*0gvW1zf_=Xz%dt%5H&IbfAZG6a!3lcT`z!CXV zq6M#R1mPcA1p~a_TNk=U4drLLtj+!30u&)Gr}mMrli@d0Hw?xd6D9|wtADb zQwz6(hVt^cf1prv!Ij9kb~!^P@<>BH~~n1--+d?Mp88SR%P@RWEQ!92qrBS7EY!@R((W4Io_;FTd<_94#n{!TSnDP60@Y z(z0Qr0!+;SuEap`mQC;U?-(~if6GUjOq#qBUx&Mu6%{z`qduVeOv5-}uJ&@a@J7r0 z=IcVYk8Q<#ea@hJG2x~099>aPA~5^7Vk`v35M6!vBEzaGP4MTReXIp&3K8dm;^;SO zB-@XmdYF@l5WD`!oKL`vcZ48ro_+(#x$ym~X|yGv8oPTDcMO^#vzID%F?bZz3^@JH zIPC-}4-r4Aa8v=*?rBi&Uk=a6-?~jLU^8#gztiX3mR?99KcdeEppYiQQS8ZF%}OZp%&BCdHXS3Vld!k7U}ABC@+oD8E8pTSw& z(%{?+?Fb%c%lPxLXoF4aCNG9*hh~{!@Cv71tj5R8cfBoO{DozNLC87pau?uPo z)0?-h*yb2ohg~uedRcMs#$ENwQ6x-NFZM&VUcUD-Wq*udA$VoJ<-hlL?j-ealUJS& zPoSnEfII)qOD&FvwT(WmLhJ|`Ggq<@fQ{_Nsr}cuJZQ1a( z@$1yn-X4pyz(_+SU0cwD?`=ncZg_;~cPoC@?^@7r7P2lej*K;j%xpJDy#c zHnjU~`F`=PPY6$jPB9$)Jsw26-7xu~npEXLkjvBW{j| zHFSDwbOnP^)7eKlJ(y9=78gQ%uQhl3W275t6jz)J6tP<2Mm_em3HFK`a&%L32Mv}Y zW+Tt!GPKo*6YGf96N!YWh`{I3)5*0?vgV6WQ726?sC<(ATelEG(T)_Wq{Ul>y7pWb zoWajMuye2cEO1AIpf$xrmK+_v5Y?mJcJl5cC=G(QugN=AL!M^EkT^q76VFKAYUb=$ zBwKx8=ZGMtmom&wL&yaaKFUG#e*@cxe6?`aX__4V9foMqi}iQv-1s=NuQ^`*gTHDL zxQ+&OJ_$QcuGlICK2rX(Fh_LdRq_@qdBQPHG~%E2y`mn)8}XQ*{RVp(JIcrmb~T1s z5N&BBcR)FIT|QWB9^&4Z(}U(^);08uQ*DlH)Dof+>lhwn9=M(LrU_#xCEsZPCj%C& zK9vpS$=-9P3o&V40zxA*B>J;r_Eo?L8&4cjo)s3kZGdG^{TqIsiLO(im?Vr`Dsou; zQap_HuTp^1!qa@<<@~~(|NEW&n{&z;%@+M@<&Ia5?W0}DmA+tNJ)58tZs@9;a%T1i zDCLeQZfij0q#;Mc^WddDFC5*7GptG^e|XVHqLg5aa%Wv_#RNUj#YTig<>Hn_8@%`WfZcI*PgVzyWskz%GFLmr$2P%b>cFxV z9(9)S9dol~8zpwpVA8^gHa4TVGbrVQe5H{Fo!Of0T*AB-Jl}DoIz)2Ov#hnn-6xWQ?Z9VR}m5w z3rsC5>TNRptRi!Jt2&b}*>Jhxmt!C>Zz_gKUpAGl^D5F^3QQHaF~h_iN;<<- z!y?1GKaN~UJWipU(_2Li2No=nCiN!Hbas-&DIq8jh<=plOKGzY2rxjI&>(SAqFGc| zwx8Ipczm0ZIJ2eK} z+ol8~Sk)ahC&b;*DJ@GEs&+q?fUmJPm-*CpKgawc_R07ZUIsGS2vr;5lxR7#X?JLX zug!XFx?3sM2q-R~OAJ3(OU>&<+gT7eio)@=ljAQ0rFkIc( z*Cozvzfx8|Fs_>c(6pCb1;ZzbzfNPzP&CJ5r5YD~z88GBaF~`Zzku>-k=2zSTq8!i zsIg0R)RL|h7>_?L9!J-AhCQ(R?eF}{5TsPansx1~WnfDOHV}51xM56r%?+?|TQY(` zU5){T=G(uGwNpcwtNJrW*pfnK6B#B*m+q;^vi8;G=-g8lzEho@L#K3eekPQd+FFdg zLl@T`d`{I)q&t_m-Z)s~Sw=lQmbH5dHz7`$AF5Y9WqkDcq-3WrFfWLdpJomS$n)MO z5SE-MXkdB9kpd&8LQ^`L>EahISnjoFJfufN8^Rw-H)z~VrYpeLY$%I}Kexg< zdtGpEWm^u@=mLcNkbaD!{95wpDzY4jvfh=DzFEvutgoK2`;mb$yK#{%T-CQ@kaYo_ z#>#L^$*UZshqH=P6gwua90W7E`vcZAm>sIn8MMiA8Gj%HO@eHp7>}aE=TKTUdS>1# z4r$&9cr5HUosrgq2;JzHlEvgn8trhgi!&*(t&MM&aAwtngj0sL&0Q5KqZjljYCSvi z)pd@2f8iZ@NF|sJ-AnPVos1o&U`W}Kq)~$|yW(c|c>W#f^dw$J3e!9oyMQl(2_{%7 zbAjUZRR~d+zM2fvFm+!~y}@^n28^%n(|oDK=Nste77ALvHCy)O2++kI^u9+aKGY;%@TtkBJLcReJ?WuRHkW73^5;9e5D|t5zlEq9obOL;o9vYoq=$Jr`{YCm%^SgsbcGn^S1Q!qu62shB;M9Lr% zTArL{IatrCTL##rn;y4<076Ek@Wls1ixTv`tAo$Q5`0>|=9Y*4vkC9tNvme=g@89OUqo}D?p#UhiQejMn<~;&*J(bsroSX zJ0F-Gz^{>c9;n@Sk!6z%{mX?(iw_9UERkiv4R`Tpf&g6T7~=a-)4>M z<$+GRLqogJ?0ZyXl+nS%Rxyfv#)kb4$b z@WoWGif8)$)@zYa64A93Ksn88^jVgME5u0tp;;+~+CB+!;-J?}>sP0*tK0Xjd+BR- zV5feh-p*1)M_@8?>s}BocA~LfA+t+smYRRfOz&6GGKX4Ci&sa8UFoX#^&)9X*`zb7 z`t#0r@n;d$_O=bE8-8e$?1^_->dxfK#arR=Cs|)u%4Z7<(TIgM3 zDLLFM+6s8TjIZ`eq<`4FUY1-gWuOIm@0g2|MefiWv%XGLiKZ>Ly2;g<3yg)DOx9q3 z=s^4bwtn`0m@MvH>bkFPPtCyz>P4~wm2a^j#y0@$%6uIOyMMx8BAgS&>+$8BqPdg% zSv|F0p;W2sDmo6n?Jp}%=bOt z9%2SXB6~=J`Zk!;ZsyLhS10p-C~*o zlmq#KxyF}pSwVs8vc-4tB6_dxw@|5NM^tr%DqY`$x*#_2ZMW{lvC~Axb4P^J+hLV- zoao49E<$*jith#x4kRTIh}HW83F_uPs4M8S zklS;W%$;h)>M0Vu&EI%Ut+yUh(Z506ok1QXbzh?=l`AlR@98CE`=1(^Boy8(wxIy7 z5IV4Td>U2!!xg65luE|^;uBICvO?`rwGW_G`G;hsE|i%X@^?8jVl-LzEjIJ9jcyaP zku21$L~urC6&SW7dU?#c#T;Q`I_i$r^;(6JRZrA`md5yV@F*s>;@uT8GFXQ1j*9<= z!A?gxL$fhU4n{R%_93Bh`ShO~)qnd)7(HkyQ+u z^SJW60z&M&g^?AHY>W$^7t%)?LZDLh!VaaJI2gObvHD#5n_OC%TGm(#ddg@*PLTA@ zd)tXwod~0QkgQoDJq^mYu6jC%n|CK*^E2<8f4sR&J#m$x=N;(_*D-2RNR2Y6j4#xV z%KiXDu%c?uhPE#RX|I;#A+1ll?a_kokPfmvVm}aP)ui3E?E<=1Be$AH9&NE8I5nC^ zOs37o6%m&K&4^68|GC=cy%f^)_d(3+L@}1CEd>6(<}&T^v)dz~53RolB1RjbQs!_m zdW4Un*&Ef!<<>gIWgvgs#=IStm)OVzb83IrVoGhl|^7tXNT$*^{;P6?in z(nkCBH@_C@x4g=bs^mQr!B5AucEgJK3s;fKS9Y@Xwu>KPa(W6lB0e)XFsX|mBh|AU znyf3o*JRu-uRT?(XI3TF5;G2d-^1k~|%VVv_}2cP#Yz|?7yX1)CY`3msy=eR1gPWdey)EYghJa+|eU*7a`zZj(&&=~z#+UJeaMvyzFD3XA7~25_j-Yre7QSD6pT6))gPvJ6oFRQ%@rfWQY(sXO|5e zj4gLAmvEzxCe1f#bcy=nzueUbzWu*)(M|Nt%$T6-kc#r4a!J@3HKja;RDjc0g|hXr ztM#cW%xLe$z)@bU1|4MuOx@I?m=GQ;4Tv?dv36^~X?n1rJ<%vgENOeJ;o5vd$w507 zix|g+M7T?JkKFgE_a%#xOa6svT_<$UBW)rDd93HKbQQl$f3u|X!&MPzSKPhX$@?(Y zCZngk;3fubTXCw*D!--x1lcJ>bK0l4X4n=iZ9uOqvU&ej5a^w6vy!1@*9hHVJEE0@ zA6~cA!PxCZfPzS)WTTf8Swe?*G|)PSv*A5>IV%9nG` zSVZ>HN)~u^sYz-Ll~IyYQb&0_|0E^x$9=5Fhh{~Py!&BNHy5Ki&CJ32?@~0Q zwHy^oxG6Ik7xzL$eF zTk{9=(OSKZpx2Bd0W(WzS<@G>7?`{iR#U;@!o}_;y?laX8=_r1-SX4y;g8c?`2|n1 z6|EsirHk&p2OTFIo`y+jT}NQQI1pMNbiGEeX@ud|N_0|kC*9co*XuHL4z{^SPv~B8 zb-&BD_Q0K43WM?KECI;4HVH4|r&iC9l@3V>80}FJ?%G4ZzB^;^lV1EZ6IZhHIGp}XXA{~5<)Uy;LN3V>jrab4jEeNc?FLQp)B7s;|Go_}bF4&|fG_y%?q zIQPsP(}iEY-yF2HW+(>O+2_Yf5&GJ8W8A3=d29_xvtZX@ z?FGKA0VIEPPoMJ(U&hpUJI_o5NAqM>DeUGotq0u?X4t8y*;!4*T=V8@E~gF#w2)x& z?T>rqM=a1RA{Kp1FB*VcG~Pjv%1dV*LjwIm2sWk^svS$-BdK6~V$o3=r&Ef#a=GAV z#vaVR_JS5Y2ARbN27ywE_F?8ftiM}Z(ch_Lg@v;P0@+jW%FTIwy|&%w&ti<3t^Vw_ z4!XXIzB)Liots)2487?|0Y_IN&oFxTy6?L$Gw{o5!ZLx)anw3BXM$qj&{t)!rw?cn zO0E(5Z}Fe#I~47U*>& z^@y{+g5?b|f62-8i=4iyW~t$rJ^1bV=rRXdqeh=ebK=bUuO>+(sh#uaouAlFI{$Xu z8Rt3Ka>A&dy%hO(HXNxE$D}5DeGL`X#g|v%&;7T4Qrb$I*^4fMV4A`ccKrG=3%V!jkbQSJvoBxS+Cyl-LX63Eek zX{C+g+XXfM<@%_VQAGzq%3%d=PWA(+u33oFhD#}yqAa9z{}f zZ!J@bh858<yALJKTcBAUOYDu`G0LCI5~Fj0%oy6)K@lJ%CX-#y{cOfS(TBh?2v*WoC4j2PsA!^}diB zQ8a58nl{NoQf5boT&fIY3%~V+z&Wsuy&efDBuk35X+J=|qH&M` zS)k`MfE;maImABoMRXyVPcbJrMmY(fWBh3TB8EeJS7vz=F?)kI1)^yNV&(T1C1>7N2c;UK|2Gvttmq|D)wc5NHf-S&ZqA&tTk_ug%4BL zZB4slmO*p`*gpJvCybl)HAR#zRN#R_ZE;Q7by&f345nOSEU+&w4l*Qor7$~2Dh z4>{-=;_@o)Qi|H;-LmSGdf0?|ZA`yM6X`&Dg5*P{XB(y(+J#aFv&@S^IN`o)Ob1mP zhZl{^(t`FV!T4QmxZ&NZLi9u0PZzgd5mLhL;>$e9gp(GTX^<-zWWe_-E@>f3O1(7g z6$h=$u+Z1SUcghTH}u&rta0fY8=Bk*YM60z3}sQN6s^KWDtun%_ANn_ zIE=dClsMJzZhcocNYPeoOHF2WJ0vQ;pDDrYcX-~T+1&O1yX7FMkezIo1X`HS$%zK! zMQd7c?=cUS8beIy(RT}UEbEM!!Z2U9?#6(^-H&dC4`P6M<9v@rREWm|CeR^Zknga5 ziMOba8* zHm_Qy<8kINyF8aAvc@68@5OBgX+xeSib#*uax8n(tYky7+;3}X3x$K=w8?X)W%Rv+ zdCZ9ESsOMU2=7DJ6eF@?d1Hq~P&LszS}pHF1(=6{_Sk{vp%??Hyu$RFd0#yvX`yYh z8!Pz|hly^54Ki`>UD>Sqv#Su}BegRJi70}^!ah*H&nV!0WXsv)6%Yb7m-7taY)Nq~ z)XO)EjxEowx%p%Jh1}$_P{X(u@6THPf zCA;h_G2FEQ*IqpMXUr}u>2L<^VhpS0&>zQLo7Z&|6Y5(nU`Oo?IU)3z?BbKTndH5} zHeedFe6rrcGx;8$+vDGrAH6B|_jl}6|HIEcbfwNeL;LTT1*tzXgAoOUXFdq-UL{S3 zI~op5OirYeH(IWpnc?RKP6`e0j%&Qvsj7hEYnt){oV`+{p#@(!!4B850lju7e^I87 zEX2syxwxGm`bl{?&~{p(h&sXss%onBvS-KT3Ut32q=q`jpTaA<{={Vd!eJ2v{Bxb{8)c9TtDsqqtePx@~QV?PPl=openng)>k3 z$AH$Gnl?Y0*5BdOgk~5eZFJ_qdm#G`YjtJ2MX^GAa)!3}MRd%6hzq+Y#>qXxwf{Jv zYRi9&VV9Hf&ri@76t4O?+?5Io?;B9QB79hkyzEkQvzv`!{N}yJAs@C+Pq*wdFx9tu z<8JA>h$XwhY*E2v2sP|4F6wj&1pTh%w|K~VfzPMNb!%;El zY6}+nMs9Pacc@cR-iLK29bL*EYXTmPuMs5s7uq&8-iLjamPsxvWdOU?Qik{A+sk1r z71SIe%U#VkV&~wU1FTd(rkzp`9nx?&?Mpw$JguIphADk7&&2Wi7i#coM0CF(Pmp6( z5F+Fxv6gsfvC5`F{k*1S+;yaNpjQNTEO?wKyEIuuUBt)@YtmndeadxeL_g!6k{8{w z`E|zV_4`9=)+3<QBpUd~7dHw{dxz z!$E4JTiY*2uh;ZoAZO5rzn~8$FK6B#o4v=uL|S}DUfSO)d+i60T{ynSzWi2ODuU+e zs)Nwf;WV2#p@?cOe6$ONoTnP8Hm0#YQ8JlP)NN)`tC|Yow4xdU&DBHQ&ox$=yAeMOHa1nuImME z8TL;R7BRIZX-JfU#gqN2SpU?Mu;Ppp#uw2ek8-)M8?7p?@89?1Xe=j98DX8PCiX;7w;C( zn#3>qOooL3m1CgL6WGO37d%JDmC>&jukxD3zh}bwl1VTW2g$p`>+>J-A4!}HS3W05 zbLS^7Dg@XhJw4|1$Am=ET?maIe!l;-A*AmTy3B#Q8AyiQe(e8^3M~9`@9?MYt48;9 zAYdhl<8z3+D|+jPG|%qkQ2f)e7U?gkj(vTFjGVc#rGi8JA^(=?*JPmmR_e;{)L|}HCnxF zFg`P`6DEnbOTo|Yo#JGp8iY4+0E@hrF8p&A#4Qy6^MWc*T5#)Zl+4QppwV5C;aNS( z5i~B^u?Lx>VXzg&O@*K3TZFxdEogknL5OBx*aSRz+4iC-Yw?^0VwOx(=5I#8m+pm1 zP$1257-%Tras}J8SCv6m@xvlc<0NCFSo>4+ayo{9lns zG^yP=n55GiSSvSVu-LMoFOVvw5$6#r;wzBQ`SWXAK)XP`38Tl)bZQfMMS+^G3Wd|+ zKku>XgmhmWXpuwKwN(GX>JG754GeU(@;Q`tl;>$>QkJv#B$JiF;;rp3nZC@G5xQkC3PczU`7D>O=Qr2XQnvH91>cezKL*oOV!4Nsvqg%#}1;4beKkj{{d zu$*~UA16xRF45h@^~e?EZ?PRYC1=H8B{~aue_e^`O_%F=CJ*!h0VoWzbE#q$6NK<^ z4ba>=$&Fpl?{wj~u4ITBvhOpEv)2s&Oq*H^<&K1BVImop+tOc}s=p|`9vw$Ydf~YW z&9~gxd}r!?kK+o?yi1sv=XGkI5L9YNc-CDTxVghC%eYN{Y;Q2x$R&rdM*1Q?UW-?; zp1{lhiPf2ajh3R8(wpQ_j@%omWK9kOD0MZ?P0(> z(!FiMP5J%&|br&!hPg-up9Pj!wGkao@mWZywm=epa>IkPu52F@g77>Gc)vkhF zWH(yRpAR1yrwG=aiDwlgo%4Fl>Wym7!)KP1Og5DE|p(ZJ^F=- z*t4s#(`*eHl)b%*)c#n}uZcP>%Aq&4!*`72sb;o5Z=Opukyk{LEZPEVS`lvW#{(D{ zOC-q+6R{xuhyPM^_&21Rs+4NgI_9C|fz|#fl>83`X^uBD6Z+lPf)o8WdRKRJ)>5VyUT;aNc83b<5nERoGS6|1pHZA7GjcijcMNTuPMo^6o@=`t zn6wl(6+Qu$)47c8<^d+MZr<>PI9kt^$9jHz;TLb>=G0Jyj6o&N-f_?LW$DYeym!5s z*!4{5TKUVnkjsSJ#VyYaqu#u?c5MNuv1j|3pAmE{WT6tK+HHUK)-`^rk0f}DN*s_f z(kZnfErCX}HI`70s(6|b{#)<+^p*8|+H-FO7qKr2;0%uIkQm+nd9KEt4UO<4O8z+y zZYxBQ+_;G%801Q+VajPlDoRxM+Pgh6?7!{*sC@65`$K4I=AN`DV;m1mbtUpE<>Xau z0R#{KWoIT-K@}E9Mfd({BWo;SE(3qaSn9Vv+qCfZC>{qUoC7w_$lBPd z7=Z2`=0|fzSFmWpnc+e@O)nqqbU>-OnRM@`ea(RQT_~6QeN6^{9X?G9X3M`zGDK@b zom{i+y5xKLSjPEl9{pfwB@n}I=BX<$ajzmE6Zbl2jaJ(#B0d}nqOwo9zx19*gl{aF1hDtD#Fl z$qi(8%Nh=>V znKZ7sP@gS`m)kx)<{@hq{MV`O2Fe!jDNN&u04t&XY6X*fxAV@#-)O!iTK?ZI9B1_= zUtvRDV8|FJ&1U;b4374p;b+0H$r~Hub)CFT96B@xq}1HClNODk6WTQX)YsLx3+6n3 zcZr|J&|EeZtRqiUn|@DpMol|^_D2zxn!iEBD%NHyp#%Wn?RU@x};tcZq@R%oBR94 z0(UV+L+oDf8^&=9A>bmZ|M(vnA(fF>g8@8)hV$yEifc+^g0^(NGXrf$vivUGB~s^w zAn#Pi@N<~W5_3i|T7|}Xz`q0<4@vLF;L-Np;M-dFv?o-b+JnOyEk;n*J^ee(qs-Gn zc=XBm*{=5`#+xxm`$8kJ+BZt^%GJ9o3k7b;JMUin9GE__cGqdUZgO+Q`x-|-cD&~c z+j;kTSV`a9m&!~*g`V|?hTzBzXp$(5rIk@W9f*2*~9K0MV5IC~>T`!4|=|AYR)b7Fa>1wBQ&g^5+$OUHrUU^8gs{Tu9-(vh` z`GLw|2JYeL%l_EFN@`rT%n0}D>CS-RbG~LBy>q^2!KVM%wyB6ONW^_rlAXs_k*cvk zTKAt}_Wv4Sw^YI=e*kxrf6d-NyXiRP!V8?mZ>-TSDiYU@!FqJn##oSyL0vA9DPqXU zC5|*GHJa1uNJ%q{OoimtaivIVBGcR#crev+v-UYxZm|_`@N+!Sret0<4?z3+Wv0ScNXAG$LIc_S}Kik^Ug9?daKX-4C ztI>?yf{_`tbHw=0P7*xxS04;vwpG>jsT9?Z$ZbA7|1u3=3EzlAgs&)vSTw%WI41Zq z<6B(Fwaq4KWv^v#ZeE5Pk!eCM>xzu4T#jt$(vS~&ROg#b^&T(AD5mtmbrF5#JYK(P zhZIvOx5Fdd+DVWRK6478*&WH`O7sJsd(qT6TynVb@NsMsY=}N)va}G*%)7ETqh0 z=+Yn%Ys9HZrw1&GH4&apk#3i>?xKsNErHoxdH3FJXBYqbYh!&Fwxuq1dKUoSOTpW0 ze8rISBzyo?-}esEPi{8Kqe!~RjYQIh&8)pXpv_j?a5#q<$C`hCPe&NShW3Q?7qM}a z(SP&SVVE~Hq^r&#_6UkIba z3@*(Na415!%1qkAEW--tDM)`E_tmN*Cbb{PBqd*Ik$l9J0wBxHglA6328JVpQD@PkEbE3%vB7arh-sp^4+-%K&R&=4S1#5vY zwqaWTRR~RlzGUl{%=9CP-vD|nX3j&c6~1_(rBc<8*hs*MuBiV-qyU2xY)m&)-~d)w z8Dh%a6a3`g>0+z&%$xMx9#NC&he-qY6wJ$fNxt`eQ`%ifTvO>tdu}D$&KJF@+xf7* zXw{Fp*U@`R9ShII%@x=GgG~AV{%um4FCeb#7d{L5--j>)P_VxA^7IZSF)+&HDG5`N z#_3?%HOwp=$O(Xuce)SYVNHPB(Oa_rUVroR@AWMm5?nf^=8H66)ss#Jy`Sy6uU*u0 z`sRam1ZB5w}8SM0@=5D~@pFB`!;?{#c zXC#E(xxw*8ZyTJv56OJQ4>eE=z?oypTYvIOZH zXRRyEf4o`8>j%DQk!-uieS(Fpt_?Nw9|~V0gWV0fiJ-pme|}!#b5xap;Hd`lU(YwK z68%ld&+tftU^Y)xRo(;Cz&XI4HTkq1_i!Xi?exC7@Lq5EM9KX6Q$a|AnmyR!5Mmx~ z1ri4V|MgA)@((e|Q(-(j_kTP#Qu!OZ^`z;1-14HgbOnPLj)2uMbXs(7y4Qc?vRmQu zHiPR1pe>EeQY^T^$-7$9Rav2(6^037dnR(wp zsLy(R`Ud#2ZYxEVoR`iM;J;3L{loAb7u}i@TqNvL#?y#q?d9%^jtK6om2t(rT#AVI zupOJ-=98Z-1ZmuZJudJXGRt=(!UdJv785Adtmav6TmAPB{>%!d$`jGcyRp1Zg;*Nf zvB!2O(ePc+)q|x$-zN7W)$5*|{%TN34p6^e9Y#!OZptwa1uzKvC|Y@6G-7A$xsHIL z;Q#dyDM^#$q;iFNJv>c6-l$XtOQ<)k!FG~-M8AsdW}g!ccgPAR_i>Z`Q~48+J2i=* z0VtH7QZZE!7`zhCsf~}*2rm;ua)YZX4YNYM9!0svCGU<9iS`5PRYlZ3krk8YjB8zv zri?UEaR*slo&KzmZk(t!Oyo;W6z>cgsu2LvOyoHy2}(2?^?IcB17@dHZzZGjunDJny~X|46{voy48u@{^eP^-4$j|5 z2v!xNxqO@u^a)@5VN9E&3Fv&jDn!S!&(gJ;M^7!CrN^Z`w9)+G_*1}M4%muya^-M4 z(duki&*fC;oZDo%_M69mkU8MZH+-7SI*AZi4pJn}B7Dw{b|1C?-q>fQlsZR<%yd

a?#d&=x4_fIsI=Q=mFO%b@0K=)uhNeX zwoD6*mH)@(%C1IoJ_N%5z89lc;%{pkK(FE&-qwxZD9$2&F^?ZfYdaQwsrRxrGQjyW zdZ_8@VWwS&lf#hj==CaY-7nvOsp?>Uiysu*_`~nD|BpvZj!;Er**==-ZUyiH@~7qq zL>ev7$<`pAAC$qu#V+XaTQ|NhW3ODsd9KlO$Pe+6d?%MN)>Q>5?M7x~q{X4e_TAm{ zNwMEy2HBvIF@i_=GF@t+S&qJdqc;meC`N~={qAU%%vqA|T}k*14C=p!x0Dd^q^4iM z^_A`J2;zzsT$=a`!QYi4mx)wo3`)boZC#jjVK#zYV*}%#Mr(~E{F1-A$E^bS;!5Wj z(PsEiVhY5D>@;&RvTjC02{Q8`J_{{PQD@{5X_a{Au26w_-tvpZImXx}OdS~_@zK2z zOg;Wh^wcXPQ47QY6ul$r->FK4ljX--eB2$2FltRZYWW2ENIt>>27mWYij`Fx-vX9M zhnzx~W)OI!+Rp(4UB3$Fg9B2VH9a|PrE>V9Alfv8IhJHc5Fyc!&{BAqfK$@Fp+O=U zMciW&%FL~(-|+vRFC>_|i9%JG9e-bSz+t|`B&o4AGowmsC&pX@Tg^E9=4Q_aXLt@Cnm4)C6}(g5EOA=t{%l1 zgi+!(WwgB7gnj})S{g4(^IwW>cwO!rPt0U(cr`S6v*jCJ)@u>ibm-pSf-W}>C1j4k zQYbXbvpmU4X&mz?Szn+{L>p-1j7!}r{dTOaEat9uI{oMQ?Eqz=FL0&NeB6w`M{K={ z>TGz_BU!i`jGr;tg{P(KG5-6zbS)g2A7|uiN;|;a{ywCsil4}OS^Rh)Q*G27n~n1@6-s*sM~DMDhzbucV~iYHzggvE zoAEnl^exsd2O8nXFzs18JP*oAUX2{bh+9Gbb*R6<$$lC!_$j&n`#`Zgnl`8kBxEw} zx4!;EgwM%Wc}(BS2LGq7d@$CQWV)X;Y}zONpb5~0&wJ`BRy$(wx}3F?*V&1*5^!#J zYFHlVOSCn9+{(~zVXUIr6n|5g2Kt!^@H7kvM6TK4@QUw-9wT&IU@C5+3er)px-M-QMDD;IIZ`3Gbis;;7JH@5n=qiw+ zS41RSp|| ziJ+G28jx_5xi<%>i0}L$h>KU9t42KmT28sAljl(oG$1;WxB3^q%1y?azg+rB4_5Ab zmn`f_8vr=4>5iFYcjhtt)}FVtUMTR3Z{l%~%g%2+R-QzMxJ3^l4( zrM!AjAb@w|AK3+nr?lIe?*HRqs<-ovic*=kIcfsJY*xKN;nlIsWT_a((4_P`Hn|Pz z2I@SKy-1Xj(>;iEd_YnFT!10Lnin)1;n+RcGWyZ~zsv#fk5u}XNKG8D|C||+L@T?(lP83F~opcoZ*7~}70J4?z)9Znr>vjoL2<8w#V_7(5 zu)D9`ajPG2Z+^v70WgYh?ww7}NZ5moPk<<7p!6GOq;kAAw28;~73eq=Vb=!v(Sz~9 zUZp6ssvl8J(^^LBXCkqDnG?jR6Z6dlA1xa?r)#nH-|61`21e&@mc?$P}o%bZeKq@w%8?C4WG8w-~_i|=7Au$v-pRWTJUf3Q?P*6=o zP5P_vj zNtmFFtsifE64Ic0wwk8}Cs(5r(FUev?ww_kzh}&|6axof!zyt|C?DN6zpq%#7l#g6 zRF?-vigxf!H#%>s&#w{G!EEvWZ8wM)U=6}}vGlElIS-(}|0XOxAuRtG6GT#a$U}88 z*0H)|P23k?ae8OfDY4ir@xn)@1ve^fAxZ&TOSCuW<6Dr#k@5eb=@AQHUy6v>a!{oY z0%%M~Q-X*QbWIbS%0j$QtuX{I;RkNYYx0%+6?|ANO-QvxA?9Z1wVShK4=8*zU{q#6 zkx>>$if&ovenO-*600gerR|?lMj@9G0upJqPz8z(;ezpGC73z%7?M}fhI{pZwnc}5 zQGz+n3e6G;fyF$JwAZpwj*|ymU)`8_HCr)OV6Yj$=6(5$$Xfn(f$$s6dkRnGSh@zsv@S%gVICbwsXE1#!g-cWURR}zjA%8ccFR=)9${cHJk)I6kSF`Am zAn@V!%dPPLugi-#Ia4Dky4b~|{&yv!vCSE9#_V>mOOez{N#1_Z69Qk39;OGn2Pjp& zCPGU44bq+-IeEp7u@yJ)tHU|S*shmJ;!fXb{&)SV3SCnt(Fg*$8nCjU{f+~syKE}g;S=Rv&|2yj4at(>J3K5=n+ zw_Y0Vca?)3#A7E3>h*enwmkK<;RL@|iYg$&x8D4J*m~=*D%Y)hSP*d`2n&#INd*L? zq)QqFix4D5L|S0cT`JNcihu$Nigb5}ARwT0hjdCwe)I7h&)&Yj@1N~;?Q>jvt>?M# z8Dow)Miz*ttaEvgWB(P;9MLs7bczHZui-93Y`So}_cRk4VO^dGysoNf2zE-Nx?g5q z80fn)f7JPoIsoB~&>X{Sch-)+frX^vm5Q4OQ|@eR{9lrmp(eEqg?cwm(zG zHqoCQ_C%SIgYKM*+5#Ds4wJL0@9V3mVn0IBbe8OrQ;yOB7fGj z{Pi@>2Gta_ps?9VDdoR)SDDl^>4^dJ@dIYXu`<-*>NMR`4HF>vC{*ezhT!|`UG?TQ z1ep4#^}y4ouhdb@iY=vJ)`gf=UYBb5bs{6peWchGTeDs00uHi=pIKlZP-=OT&N?~4 z=ZfMJY$bbtg?0+pf~SOXBP9QRE9r_jy>~6RPe)WT%|3IEW&4ByU+-bS3w77#Q1gU* z^D$cbcEHl~-g)J(_?0+f=f?ywpE*cHY@5^ARM^#8eKgzxnaE{i7CthwGphjHDJ*7yZ!bw{gH7PY01K6){fT zNZChpX-#AG2Qw)HUNLXdwu;Mg7zti57e&og2`0F`O-v0wQJs${I3l*ZJys>Sx{G81 z7DJaF3%0!{V!Hf)vR@6F(UMxh2{F%1#TAo1U$*`}zJk3IDn@e05wNRvTUA1A(ukD9 z`ci-3Uz$UFx|6B~A7!}I0Yv@Gw{YQ%6BtHX<+lrYTIo3Y^?3gklyAz91GCr>gfoI^ zygDVU2gtc^TIq&-Cx->7@4bgv#a)C{mm6dh%9w5@D*f+o`Zy3XPS=5Iv>phj{r(B{ zjY~-L1R?s)v9RrU4fRzSxxcQFlC0tWlM)05*#C;7{A()#10v76yzS^GDiO%c-A%0w z8r4d()X^Ip!gZX(=ZF(fvrV*d8rJ98VtlP<`f7h7s;cu)fR?l}JXc`+cc#+FMoB9T zf<<~Hh)wvD(ylKiK@Wsm{{9lnfB*6uax^SX1`<)& zL2dM_J>o8DzqM)U4xI`_3&NgNMSI^VK8M5uAP4{(cD8c5YY z1@(SU{nyn+6qVN?e3svH8u>l~S`%4dmGxUu*$1h4e6X^6+hmleApx7tuK$V zgj;;1KjoqIe~O03Olz=3js1Fs}6O@BYebf40`*#xLW^|8aHRt}eyaYct z>&BM6GcP(3-obzVIwKtpvYm!>(88dZh-iVo>q)=6j>`G9(dl*6gLtB6!9rLTV6N6R z-dM7Ve;qGeG=v^0T3n1X3$oR7k2_od**;cRD}#hJa;bc2Qdt{uNbrag{n5nzLbrp? z_N$EKC#|#M*n+|hbeR5HtDIWgqtfisT8qD-2zjQODk*d>~VT&$UpbClT)d)=!PA`<)_LPnT4@P zT5uoiR^?efODN{JaQkY;#ldgH{a`>cJa)7#jObblY-N4)`YRuWJg))e@R_242Q)#7 z10WoqAK{kLrga%nj@=lskoE9_J6h2f2vF%DVwrs{-E866e{Y>ntNb`PbF9TPbE0J@ zyie+ZFxo{Y-fJbZ&<3O;wS+FrxFv}3I>XxJdnf$!`{A_2+jf2z{MS!J`N$G`I}>6Z z!VlS+2nSK=AE6>YD2oRk#FGtP{T{Qw)n~v@A2nEPv)V@s>aF$&c2N!;+lw0U4JDUK zr%9ZUNPPAMG}(1FnjV0dj>_E$BBUFJ0&ec9%C+SD1^>UI)ODPygGv0IUeHs1VcwDS^igrl zsuwAIkXVPTQCrV_eH$Z&+Jdd}f6mVv88qYYPnht`fM5OHe;rvHLYzI|NNL_RXWc;< zg9xx2hGzk2nRkm)lEApiY+u38PPQY9qPs7dl!OwZY40E^vPkhcPf%xc7p=(x(m4YPDnZi%cUaYwm&=R^0r}QG3$oKe@FqU{Kff(YIr6qZ~hPt!<^P&To*Fe=rRD63p z#;o=fg##c=TorjL?|o0uXh*TDV-VzOAEZ);;qcBr?L!+O8`e%pjz%=G@?{>x8!Ibz zqeHjNXe^9e#E5=AyA~`u-7kJ1A$s>-D8^&3$f4U(E=oW(O-($kGnRp#d!#ig=z!P? z^Z0@3l+|6S%QI#>oYw_Z_SE)MdjbnpsY&!ZH|h$cR1ZsCEN4Wz>5cER9vf}XIwwZN zaY=AlY>6BV7)Oi6>&4s-Eyzj8@zhSVAN#T7IcO#p+3cKI^^ToxsxO5=_MTYaY;3Fp z!O%vG*)eBR4;Xr9Ilrr0s;JL)6%Xjwu%-YAbCzu|1ol z@%^#xO#NMvPe~35&Ys$AU-E^NP1Fw7#TcZ*($?eKD{5i9a$1wNl-|o;5r^Suau(1H zbz`dTdD7-o*Q!bmuN6@8jIvoChpg3XUk%`ND@aUe-Ul+l zUsgw-5*9hVPz zHq(tDium*{)5Hwqf?nJ)XzZA<6Y=jS6zqTS(`4iefB0P7euuNICDYQGpeIv5PC=(( zBQkV-19L6Iq7Q#D&@s7bWHrB;?0wLTNN22~YsIH~JsXTlUwz-{j-^v%TlA@@6pLE~ zbsCj^*-GAYe#y(D-#J*gnVRxFtk}YodrUG&k$3x-ID40J97SS2AH8R2(!N5nSMTYu zAC?4@ru9V}iV2o8^?3y*kq!xZruEy)!-K8vi6cbywcX2Q7=pzBkK&C9J_DUS+F126 z++P?vgDbnIOPP}T1t(`F=P}F9t`+*ZN%@5q^G0sU=0=jU%GwQfsOB1x<-4!w>P2J8 z>FeDWPyRBZ5+lMBz;JW6bq(fiOmwvd0I_{>u$1mXvrC+rxzS7BBnM8pvIMix8?V^u zac8)AEcvylU3!IPjdb=c%_nA^6H+htdR0(#kVTj;@tZPz_fM+dn+Dux)?S+@k^To& z1p6nGzNxsy;f1fl%FE}a`-KS5h;*j9n(WwVK0MX=01Ewnw;O{%Yqg+wbA}qxN^e|cm6@FUb{gu9D)DbT)oN{8lC4v^y1n%e~w1FD9&Ua7KFfU*}ga$eTDat7P(wBBdE{FUeqlq#}QPEAPB5P$|mH zd)}8Kz5-T718f!|rhmR78r%N7cQX#xb8z;Vk}D!6afGeQr2IEGU$J2O3o^NLdl`6a z#l#ag4pyxm?cW2T@XMoT*Mz+N^-uSQ6CStZcWNefD$lPAw3-eUWBj(^U5+h;nf0&W2W}$=%`HN-bj|(`1E>4_mw(3j;;X4zzqIEkfPXWHvFLn=0@e1j)=!f+F2cr z9Q@_S>z9?2qPmw{d)Xy_?tmLe2{?X@9D-c-3cTQ-9)_3v-qO;0phj@9OhAnPF&Lw6 zAAYOeK$l3p8k*#rhe}|ZJ_>abm+xr-0KJ`bT-oz8SbAZFFump}{&Kmk1f-!q(Pt;W z|5r!Bi1QK42U5@I;n?wEQj%Em5HK146wN%G*ke~|Y@lR~FAg;(1*xS!KN3{mp0X#- z;g3PBh-sjnRU{j%Um}BYO!8ZEapn}aZ}@b$zBQ*5Sw+P%7?=?L{>*vslCh7QrM;mp-Wl8G-54t0hFbU6GcxOrI zaCX3I(B)tkYW`BkzI%P#rPGmwoyjeqbn<^!+BtK`;iR({`eBYTMNB)2;nJVzWpv%V;SqCHtf;hm8HF7!KsxC?@|*Mxo`f`j)2!C&H~Ag}S+KiN$B7iwJe z7-~%?glmaR7ViP**=BQ_J2kL|CxUF3IZSc^rr&s{KhJ^M{dPQv&lbkQEa|%*>>H23 zcUg)#_;9a)n8r9NB1Cb>cKRVrOsoP`%T4D8YqR;nJoRNy9n*#Pc`W<7>qIfuF6$<7 z3`O#;P$y9IGD!^0@=Ayoiz=JPn)RiqX|IH#c&2ww@Jm8IBnir>4MjLAG_ShZ<;r0){8Rr#YYy%U(v^^imI!qmIcP(hfRn$P9|}4tIdiZ z(0a6p%PG|1E$X;rL~(bvMuhc?SxHEjIdZ9d%uI|Daf4gZT^PZSO%)ySDcF^`x zIp+s@)B07sj^+G>p1^l9CFbNFL2J*os}k%B>^W_cevLkzZC=!+Q=LWKVEY(Xsu^cC zTcEn{PT0wvlyYyZV+sm@LFJ1So%oBk*A3Ril}2)^I^z@QrHI;O_i|mM@jIusO4wu0 zhexiZhs5K@{Luo~1kLEOW_q_TVZk{jzkL@Z*)N>+0mqo`sBp#~q^(xqC~XhWWf;*+ zQ$7x#IgB2;Gwr^skokCQ4>WVJfXWdZT;xTCPAL@aQi>NKG&R<{VaQm%yUV`A-*l&aVct&}z=*2T^9w^jQNa zd+B{yYcDCbWn3tj8jK*G6!7^Nx)kSuc*U$oU@&Hac(Hu%x%2%K`e7N0a-arv#_py} z9Go!$i+ToTZ0v93swUHz>kyXU(5bh%(6!L#A3rXXp|S{ij55c3>BVh0*_J=Gr1EkP zz4M9Ai{uc0i16o`YsHu=P5X336p6Y+M%1%PaXV(MN#@Ua;#t0b))LZk(-bzK%8I|u zX7FleC}J^S);-Xjn1b_G3!i>q5m$(96ox)w*M9#?gcOag_{au#0%qOI|&qzth<_l1LoXrpj=agezMh`;aa!F3#g zAHMR{NEt{D$$h|+8xP@RWTOM`No$z@Bz|fTBjhuOoWqH{q*Fe?5$NMC_$s>sYKqG5 zBn1CSLQn!|R3f0LL(jF}6%1YY?NuegygdadefAVenueOWG1)D*NV4+ww-#q@&vN}! zFdqfgKx4SH#s)IfL}fspECYj>Bc5AkA%Qi>m>GeX8H8-Wfnd%O^YtLlK753tWSXz|*%4(VbE{m0te1f}^5Rn9 z9b>2Jy#>xb7`p_CZJ9?zo}TFdsBs#E;B}tX-$?qC+oMB%mxXNCU(Z#p*e(GkrPT`) zxDC_dOx#vEM19=Jzk>$Mg%`m=w0LO+?IIufPfZ1N*==EL5cBk*0pHc%*axB& z)w3E9y2QB6hvzmZZIsbbC=UWYJayYtncWuxwAlNK{;8$Oyspl9e}5VZn4QQ_$U5qp z(0Ugw$`NEe{x@srLg|5<&kCsZuhZk&4j?0~KEwcEB9LY;M?jUk@OGEgN8dkqmWuQ^ z*x)_}730`=ahE*AGuEBbPbdF@OQ#Rw5~vFTwVXzjcgC85iw1Kck>nNF?&JjWdY>|}=J1HW7W?x_zZ&jng5|ZTG!3>WzP?!iCi6%zDbFC86M9Bxl|c0C2e&MwhOVf6)!Y;7P$t{W~yrnd2*}HQg*iO46Nk|P0nyjxN2Wc zd;RV3HoHZrq<>$_;L#FW-i(=QF(-xn*$k6rcdJ`j;oBE(Dd}QH+!ZZ&JgxRE`|>*1 zx4EPeJ7z5%AIH3_yty#*i2p)ZfmC&^acO>apVGAjMR9Iu;)ixjK5dCITwv(U{khAh zt7~+abG=)Hq6kyx-d)!@Q5us3a|&HB0;~+)hv+%Rol}EtjX))UW97sH66@iyKlA-X*Z(jj1ft7v=ctSKy_@IUsy{1hM_w&$IdzJ~jF(F_ zl0$@N=@O$znmk(jQaTRt?W^a=F{~GSpRxFy(vimb?2B{boJ=In4W2jYJMLSHVbuGZ zV*InOUy772xQsge__Dd_a9o}y_q{r!*;zOZ>GkQzw^PJW2jFn`7=e{x>YFr#Skc%u z)Ac3z_v@a;lB+@U0!jS$AFkLwLG-RIfLt3{50}_kL-JUCDzN2tK8|AIUB9ARI?%ef4LYEBq{G$6hc>6@I~xA^px>X@B6ni{0MT7W<^{#|gksaD3f%&w zFJUVvwFWd>7m;FB$iH!R2C4y_b*_QyJZrPi1p(XeFh(w;wZ`<|T5`YDpPdr3l6gyc zj-SscpO_HwsT*_6YA#c{?8(8$>Z#z>++5&f{JPh6kNPmPxq#RI`_7_0?bgc*`?Cyz z?f48qJ+o6LuM^iqI`TJ&n@A-|(=HJ7&FRy|7F;lw?xCvYbZ1*l3{=JMQKR4bRbX=W zHQ%omskHmyPvWA<%@zbb^leDgow>P3*QzA@8i>`5nUdDGuB@(mgfLgpANSf$@DZjs z%G5thc#^&75!yif{>Id|l)wPZ>xNztE8@7ruCtPlv8 zsGF_-c}2}R%>HV9v!D0{d|2yIMrJAetpry(?Tb3_1mJIvxnl7%BFXU3twSS zIx!WX)W(o_sPVeTsU6rViE3?vPgE>b%c7knUfs$Hd*!5G+;TqHxjWP9^ZO^;9Wb>v za6kf$i7KAB2(YAME2FNcJ_UI#O6)-I|NJ4{hjxZObi{d`BOhTNA%ka&dCCF`{v*JB zf+a-o+?m)Xu0fxSEcUXqSygWJ`o%6Mwh+;Otchz(X8ZF&e?E^MTSW;G{2BuO%}?J! z2{}dzEM}IOIuqfcfD=taL)PW(t8`()z6kGx#u_{mOshR;hDy2fzg!;Gy;(rvIkTN^!)-1?yIqOSBQJ&-_9L7d5oYsk}1U|s^}TT8k}M zY8Y_gAD1dOZ%R2!X&1R`YZuw^&(mWggn)8b z5gE~eS+&sU{iqezhEdCZolS41uRl?*?V6}O?M8s2wDjPW`TIn58!0xlUl2)@LKrH8JJI}Kc>Rn_{(KjLv)G@~6G8uDA7bbNq2#V%Gr3K; z17aXsJ?Qk%d+zP?Jc%J>+O0j&@yLnv-{BOV?i*rk%SM75KLByCKKE_qg|zo_ZVZrK zZ&mD7KqKmz%d9}`i5Q=jF zQ+v2Tg?}HF$^@@R;P%p8bDX2yW~s7U*F_p7UyQxQk!!#BGM6BQKGJDO-v)X<93&J<3Eg zu~fpCZADRP?}i1@-G_fYBcHc&%ANWez<1ENglP z<{20y6FVy#Hx~XK0O62vVM8^P(Pw|pc_N@)(s<&c^fQg!i(>*DKIce=tnzh%M~@{Fl%Ox8eD9~&h+2W*DvTq@ES7CuYO|)X0DVK1l!J`+F$tOE7;m$lVjARTaCYC?oxCB zE;@=gGOjuG09zbfyXo|oVII48;En0~*NeuIGn|79Ln6i`V=2ss^XxnvvgDQlduDV^ z$VQ(jC$+c~!u)4m89MdPyG&O+;grTxwVOQV_bXhE&<^p$(S>2&;>m&RR&CVWiXCGn@6VqrxO!X()|NbciMA#sF{z2C$fNU5fV)(^huH?SXo9SnLp*IQckmS`K z4e***B;Fj&@OZ5^q=KzoRrAeZn$N%8^MG14+wb}H|E)o!@sxw;FmWcd=EkUV@-^TY z9Fae}fR-k%Im0K;l36nO%yDNa=+A*NMBQihPD_L?a0u>z{-{MK{BseeTyK|7FI{KC zIXc>@E{7@Q0OC$IKu#0DSP=Ag5XJGa@mSPd0ZAG}dO8On#$N8_c z2{z;yE2yufz76^K0PF zSOVna-h1GsiCos+HjKdkj-n46@fCm)Mi82`lA;_kJ8c7X&d)^($CuLAiaT7_FoBc^u@R^Vy5NA#VqdeXZ-I~*{ zTsyG0PD6RNw`3-@2&{;>Bc+0OdF)92qE4>r#j*f)a`s8TzL*`5^Q$fWtMqw8``fm9 z2)uqqkoo@+_(hj1N3q2Oe@)y*{r1R)?}9;edzPb2(*dZ~y zaQUCpT_raTiG0V^k|cC=3zFetxwy7(K9b)NGC5!gadcl-D6!8X8UCdA5=IEy(jyP) zV385ESop|4-_a))T{8bZei*C>7ZK{96BsG~)|Y)Z%e|(A0|tu5P6!cNQ>7CC5ayHp zWdEGg-#ybCc^plsVv7OUi+fxpKU_w9@S^(Q-Ag35K6$DOpvl#QtNqe^==~%p*CTVn3#%E9X>fe-#?j9?ua(w3)uXwB20Q= z*l-~dp^o+V^Eq$s;%Jwiv3md{b0Md6GHEwtPbw478n{YljPp7EeXfyjJv$GY(Ph6Bb+j{xS`H%))(EZTsWR-;y2`2z0>U&53FRBR$c5N1;CERk0cFDq5HY#P zeI!i(edxSTOg%%QkpVDmtcLqwHF!KLVOtE2+aQGHky)wZU!I+z@2Z2502VyH^E1P) zTo`u8%Ao7WOv*Ph7yk2hT}ga^U{pLh!kwB5p|Y60r(gZ4F@6P^rgm)t!03n3mZ};5 z#A@Wk6Txyz2N=b*^7JlF+`rp>6TJ&LX14e-0<4dKlkN1m+kKVsjelycNpo4$b)2ii89KZaJcgbV`kevHoU-dk`;oYmbo4nd?%Ao5m6Sw^rcP*wWaPyxsp%Ts5O5NyGAYWfCY&hT_$i zf?}gG>jZX%^6~jK7-2XYcO!`%Gv?tGB(t0xeCaV57z|3gsQHHbpB~l!S-0<%w>1Ig z9{ts1=|Q)S;>Qk}piHlRj0CA+_!h(U7Yb9)a$+AJ%u0FPlP$lfXt|ot&qOEI1=c=!M97Y&PaLzw06E&P;%?rt^AXNbaHR7AjUsB z5k5lyv<4pKQmoDcs-SoSQaNZJcIVZ-7PLM`bV6gpZsbPfT2=m1s=gwAu6TAt^L?XT z0nl;A9$7Y#WJ~xwc`e~7Mg#vl=D**3`WYYKKXuZmGi6zF z;5bJeDS9^&Ae#%*EYBZ7*-&Z_EyB@|3NYfJn_?Nj(yqxla5t(;O+rPQz_7EOrp(nY zco}H8^_kGlg)V>FzqWR_cH0kK#d|)K;?Vag(^m`Bn`|SR&+=Cde�~erdnl##I@6 zRkah%WVrkQmWuf97$^Biuu8Bd6)H07@eZ?WA)81KlEX2pPM4sbX zdInHlTZ1wMH+>gc3f_ZL*-64HH4^aP<(~VRj{KF)-_W5x!GR^r<}bTkLARQ7MJ%ia zabstrk7A5R=+k(7<_ulLTNW;IN42(3g#E1Be*2EY;P}e(YSeVL%~P!*YrIinZ8b+= z9G1bzd9)om2^{-Rp0b{NcF~m9(i#&Q&SrePVeL z9{Ga(nc5vlKAY(2!My$qly>=m>*ryh0$T$XvQ6}OsoARX6J6^iCzLX5?G!QKd(EpF zv=TmaBJjaQq8yeqrkvi#;ArE;Xj=^=9!;-$&n+@+gq2rcxC^2dKiH2wVs;7H$>ke> zEWbD=YBzZ`D2O*|`@CUXF~yj5gXBZgXT`~Kgdz=@U{ANwW^j~;nAi{Da{76m3$h$& z>cwKqWL|?9)E`wA#~A;2ML~&u=X}lx2Edi#;clD7JEj%nS~iSpVO|IIxgj)OlqPFJ z{4N{!t7CMfNTNcanGMR_Ld@kwYuFPUBFiCNMmab{g>ORXDdON+4gF8lW&^vkwT|&q z*2y-BLoopo9+-DX+*px6;Raxngw9zKn&6OFy~m%=clGmMJO9``-jJ z_7@ze$b+^aX@x@Pz(juz81af+}qKgZ>x?$d1!ko6uN{cYiNjj+9aH~ zx(T=R*ao8+Sy^(o7GY(+>wU3yLA^InFaqC3bADk?jIg}}1l4b6ObXLQpPryUk93O1 ztwiqQCs&Z!|McSi%Dp9{N-&fA<vKo(J`L;OpI0;SgzOFf#2cZ(fZ@n6DPkwdY}7B5l3(dyYOy% zR3uU_7>q)nw$+d-bvUn{mGndz9l3^$gOwJ)0cF!zM65=cypl0iTZAJ0JW@yneWtRL zG^)edkvje<#hkUX5uicQ(EQ z3@FlBt5|Vx-;G#a=wUtC6E?Yf0*_*IYS?$#=FV^Tp06^e=Q|Tc$<2dYFP^kgBhZ)Y z(=nqfhELrM7_f4Wk(U^UgKA!bU$U^&^)@GoXC>WDUc0hRg*6n$l6hJLn<|5@d~_w0 zl9NxEBU2|x3K@1})GvSi5fwgh)|%sj+F-D^(Wt(AIr09Rn0eT>Sq1u!2$3m!bR_Is z`8Z{I!pO-nWq0o=f%0q|4l*cM-LKHb{rfq(;`<=TTst`CI%d?maR$V``bb=p(TP+e zk;hvG);t5RaKe-iWz~Ff{C9$&U@Up0{VJ5`kXwPfLOr1StZX+@w0!nwqQc`Lq#OkR zDoZp>77)eYosG4(L>?S!811Us326+{ZE$iC0$K4 z7GD!ZC!b#Wl>eIm4PKB3MXnMtU$Kc z3_`0=HpmP2sPVJ@vT=9})zE(COH@$ujS+Pr*Nznm#Wx0gJhwWZrF>^l-?n)9{_Q@L z)1}Z>_5NnvR7-%prV|v3?@whe4f1%d<4e8lAv1}Q7iS^vI)Z6m#!B?9>r|1nxU)~#M;%!-ez^)oxb#he^H8qLsP$V70jom3fBfWO# zCA`QH=nN-6E4ya)G=Cg?dp)gOclDMmNh)`fGFzbZ4XCkhpDShZutrEw&iy(7V7Ovq zu>~>g_cup=Ho2Q@*k!}TDUJaLY+-rLf{}=*EHE2ibO@`OFvk zihv#ai_?0mC@fYv`{Fch>9evK@izGJ(cdp;LO=N0CHqQ-T_DVP*5zkrvHMNYYf5$Y z=tN#Ld5Q6TKQ6K}NcSrkbxL_Lt31o^Nw0EkN*6)~8H)U3fpx-Ki z_C%9M&?A-hYtIBRy{~5WnX+_*T|EM&UZQhharlXl@NZS!kH~B`HNV|B`g)vKIEqVS zIWB>Ebg_CX)20+K82(?0o38wdPm<6P9ap9Zgdct5I9AFqxBMYN{=gYc8HmZh^-cu# zACMK9<1!eMdtEiy_?dJ(3@Q?)NP=0T?*!ogeU?i}Mj4$-eaTBtvE@?zN?f1dZ70%o z%dPL(o`qUb=JN|Dk*?u%z&B3i`{BiKS@wNQ$s^v-nBRjr@CZ^Syw`6xbcWv7fs%dy zbEEaGmll`5!b;RIkm~nY1aSCU$!dR|ruzXq_OZhZ{-ncZNti)%NzUj)!2G{E6v+k( zx8g;Ij6{2Kq#&%c6~uYSt!!!_;ORpyF#Im5Z+D?C&F$k2>q*=ye<-mSbot%Ak{_=u z4jzeU_I+?-jK;j4irs>A-|pKamBz{HIw97o_K;M#&LbC(uq#ISPI$+h%`^d-Sby9L;4dNR9HKfR&Z3$H#=A5ec6Z+cnGKHFCQltD+%@SRja=Q&2Y2g3hKGOSu>B?KPBXE;Wx27rP)r*;( zGlY8~sZQ@F9s6Eu9}3}wC9v~-pkx<gV;VCZk6+(7J6;~xf3X*RNFjuea*kfjIpcqI>9HkIuvG>{YgLr# zjrXIoMtzIRY{A}LGE(fAit3|fo2uov2Ln{Nbav&-`7Zs9GhruBY*B&|{$*nwbH(_f z4ov!Y&vGuw1b2NrWd!0i8Gi6Ttdgc~w*O)v-d5$#=n#LY*(quT+hNzsR}NbgB`5I+ z)pFKa>TpPfYeZ|m`P(Pq&AhDpm?S4&L=c?nSSGbwOH|<#Po}rD{!H;gk>D+gMnCWN zfIMMt7g_D)lefxsd<5)ap7m>d;(qnOYz9G}g=xc8(f}IH;kdOFjpT$kArAvtP#M%D z>RjOI{JN-OOnyNpJI4I(kp?du`-6S7DmMHicTgYY|JAiZuQ*OzbE00nrRkL>(>W}M7J*d%}^{by;OIK4EGP6g0+}67Xv{27mG}Tv5bt+ z_9{5CCln^RY8hX?qxkH~R&fNope_1*qKP9B`kV@t2E{0^w?zKd)1}xOE zBc9H3r@?(sC~L~0G8l3D)Kujibn%H`IQI6Km)9y4_j2h+@M(}v3%o>gNSg=ifMx0) z3%B$Y&$+ff8~bc8?kCJEULr`caXR-&z!L=V%r+P&KT~E{ar*|tF0+S@?+f?M22_~| z$~FXt#}w_+=C0_n`i3w+ea%L8?Lndu=EAItt?OZc(y-eG`&@>?yvE{<tu#QukhbM<%9GCH)u}V$yHy)I?#~@RJ6VwYY9J3s?WTwn`E^otfK2P3SZj zo^R-ye>X(A<8;F{0bQm>#|Z>Cu>2m#OKey-z&&4h*?0OBURuN#9=DU7oGLVI$!fcd z_~&&JOH`dBqI74fCQzkMI;LQB<@>%P}mpQt7L7>=&v<5#X?M zyk<1`P@wYo+w5J5yhyUi<=youT@Q!3`k{|oZ?my+|2yyg&mUHD-pgjZ8Z=BFtk64F z4bGu~aScVhFk~5Z8ZseC^k!7bwNl|}ftD{e+=sE)!FWbuG?39Q{d!77R+!b-e!`H) zv|^Y~``s_9qR-Hc_VcyKQX1~(L&YH?_*&l`2J{}((O>1t7)_0MuaR!@{(R)Hon zBTjrezJ{s!xfF0Z8C+D*^Se?noBQAtks?y{@+fxX`!{y7CpstZIJ|dcB@7OwI_06i>yohMD^YokQ81_sf;|IR|+YZN09#c#Tm#4@IGS!d04VqZ)vU1$D2~ zPPyY3taG00wm58J%LP&FR_59ce(qA^*AIt{gSlyoUc#i3bv?CV-`PgThjAStxZ#nq zvEc95L?rNko1z>$XGIv?OM^#gBsJ&n#8p(CVpRe2`u{6m1fq}1cyGR)bkpJD#{vOMc=1= zW=`c6w-P5&;W%1D2QyP;DSS?0%G9Y#Hv>L*!DKE)DvCMd`~X?H3`%ictEIYAhs2R@PmCBh7oyWmbL>iO6w^nw2tf zXUJkW3K23L`3uZXy#ap8fPga@@8h`AE?n8mRTDH=C!T494ZLtYg;F1^7!7%E{r$ykzgIxj1es>bEb&D zLyKBF+l13~k-U|>_7az{QpuGQ(s|mwoYXa`l-%;&P(VsX50{A)Xl1|9)Z>!rQXIdnfxa6?=FY*^e)&`NC&Je+qq=7e$kwI#47GV&dK9||5-DU zZaZf%kcP>;-@_1hjG_fIJ%P^InKp+8u60{Vb#;~u}_jken_)5!0l&Ev z{ou7VQrE6tvsT(#42--s>9YjrQ3lRSf#Ql{B-UL)JA2OfJmLjDFV1x4D(OnZ=X2Az zJR8Es5J_hTTNBk8J`wKEo6%BIswZ<%jYu6lI35h#~yZBnOR*f{z0PQc?KNRPB*^d1Ux zf=@TTVS*|q?pz>u*m71sNrdSbFDGcDts~!NKt-C`a{m1Ez7!6+UqY3XI3{Dnzo8V; z0-LfAmqPW4^?Rpol6as@JBnNC3@h5mVlL+3JCS-%;}3A3tFY^O-#t6k#Swkk%9?EJ zde6;Nx<^{ondS>I%>I^x&U-bscCOi}o!wnhNWAxI{kV9+)OH^MC7<&}KUF9RBZFus}2L`LyKMelThP<;3-0okBpsCURwFS+bRwaWuz-ihU` z*TT0H5|p%+Z2b!ASGAcmSm;I?Xj2HD`8UhI+_d2PCVj%_6s^jeTMYrc-J!0%oTdx> z9LL&JjncPbGY=9;V)thzmyn9rOct0H5|&2`9En<|Jtto#gc)79&fb*y66?}xax z^y~t(lD*$4TcC1UV})9uU(K9&@b>=cOY!DkJ zw(OXgw7b7vTJb{Hj6cyLA-^kIR|>gbR6(tdBh2m=1^W^Eoe>eaK3_r-5rC1!Cd~;B zg;*eVSWWU47~9@?>ppAbA79@N+6SOuyS zM z%$3sxQ2|(fs>^ldhZySAA?bZV)MX_~h6cf^1O;ptl!TqyMHwku7)iW~d{_m@PM-@2 zHcu=2H@7Ks0=l!lX=w_hmelX~E5g#kLmExU?_cyUxGaf}$B+KBR`+f+YHjdr1S>fz zKlNTfUdqwW_FN9Cp=W34BeX3Q>HH7E1B9C%lDjt7Z99(Vy^prfkky$JvT|;mbmEVU zQAj+C#~NNJ>q|9FTgA`X)5yZ{1aJDrqQl+0(F6_24<~)n<#l3+Tc2IN<%0^E<;b~% zH}oZ3N#50%9Xni>V|C4CQs%|J{O6=dZ6)DtrX@?{QdkbS8sLgG3jMwkIe4{1Z_ic$ z1-^2m)!@ldSD4%+c6q_7)S1Lz`Dd2cTCg3rICcu0WnalX*?ap{|Hc?4Pf)Ns-mRi` z?pv#M0)QD0ezyslUkI>Yx{40XB@<)-DcIA`fiABV5_B&3m) zIWmd0r!HWF)MaT|!ZfwQF;ec6o|((JV@OsnWK-Or7yX4s~4ZmBeskU-<(r((?(ZMuJt~%66%LwH5w+5l1s170;CL zC0^4WjHNZ3AF1OTY!za@i?!mgo~mOx?fx-b{7c3YsnC;JMI6&WRDVSr_>%XNwM~Dh zD|r?BBJHU&TS;XN_x)TuQ_JWSQfv$9Gm;tk12gypLJB@)e9BXqjD3xXixJK(xcNT1 z86exmQ4&_NWfrTGsm=PtgPelneCBETyl)>Bf33Q#96U6Jsdhq7D!M2hTJ9913PiqNX#T<8h6AEt$QB zRGD1F$BpaJ%{rpd^J~;T*>5dgOzZ0j|`{MR_(fy+a)fF zmHZQN0W~@#Kaz5Bt}>z&RFo!l9CY~RQL-fd^oijF#V_e1u?$GDT+3sJH{wR7!tdKE$e^byAD*LBPvZM&)w zN*|xaJ5`78cw&Pz7^_|&xl<2X5`_xvRi?QNrj`46WtIzMCe`7Vu|dMWg0A@RNs^;5 z(Za!R;>l2y3nIbpd8&%Y<$7{gXZ18Y%E3@pLYlGCeB}A(fMND)$lR^3KAGM4tx$EE zdQ#%%p}Us5fho)I$6)T=hw+T+0oX6+PB4Y?t(1Vka%AJ4vs8Wli!(0CnVsEflkx<* zlzUn@HyKSe&J|yd`+1bNRYicQkczb~j3;e0$s}=Gi}qKn#;B6@%fzLQq#cGS7p8ip zga~bjYIc0Ds&g3-Dz>a-K6_Dq=q+Y*=+%3Dk&8@Ew@E!ppE7BjgDXWQZ;eXwi@3%d z#<`;(9!?Nzd`S zD2hKz$Jb-yyjZevpOs|3$@%Ps(@ybs@Lf_(f~>xzngXjx88@xs*6E>&;(@0eMla`b zUKCSLmXhONNfGznOm%)l$#!80ubt&WA^{1moZTyn8!IvmhJqS{^gGc+q$f?%Kn9IbW?4WMl`m**zRq$Sr>JM#7?GR$}lIz_c6Vn&= z%WkaJ*`T{s=7eG$Zgi+c+o@h3cx!p9^RC)6JBu(1`W7Chx6JtV?H10dmNpSj*Me^< zY7p=;O~te8KQVTZ-TR*{EIL87e!*^ zimktFxTdaB^RP6|Cu_^fnnsMy_N%>=g;fhf#~Z^6Ya^FSWpw1Jnq>#_NuIbCCUU$G ziu``x)E`6GeUa2b6D9Ndb{$jy+|KZ`4RESF0i= ze78xY^NAhnCcU??k1dnxck{V`U2Os1G}RwB?u5Q`W3q8Er|fl~HM+S??@CWTN3bV+ zi=5p}k_PMf5Jj*)$q1=vCW4q+HL#$R=WxHH5)wXk4C7H8gcrf|zId4@v^e2o-6Ef8 zJVO0Vqmy{&>Y>v11=+URy6;MK-rp6g+k&g5p(ndTbxTx{JdKW{!S9|tMT2ao0*Pyw z_Z#o)%;z@M!c+f0rp_`dsxNNah%^X856uum;|$%x&`38Dk^&+PGNb}S*U;S^N=k#$ z-Ka@BO>)>tbf9;1@mZ?ftW(W1nFaepKJ?=>-P` z>KldNVs{R5LMr*)jvjWcOBRV5Sn0ssD~BK|vuX{WOq0l3rhjNrn!9Bc=!Gm(?bpui zldge%MV|H^+%)v|e}%Lb3RWSe`wB$|FP3WRWHUS_xS6Pg8d||-)^Z<%HKpTFvv{t9 zT06bB53%5y}S*oGHL_=`7~_DP`Kcnm!OO)Ia;~b+RkLHU4ZBYp@u}#LasG z$+}B0IONOE9RJMk=Rkn;O`QxeJy=`LIHI||0DOYeh_CgaA=BV z5k*bSK&(F$DdoA3$ndyqHU( zeouEs-0<1suy$Vdd+(^W@ptvnJhBa}C=Bwk3Z3FkC+Fp;jfvQ+0XAti4ny;)4I$LW9$}{s8R4N0Jw6s;sAS`2 zTaU2$ocd4uBV=xyDR z{F{1sRa*orB$kyikn_cfZx>LjZ8Ts#5O-EP|*^{IP{fo-Kaq{M%CH^Gpi3rG}Z|6{rUQ z8M}Zo?+*My&)DGJ%hiJ9BxZjtzcply7?HdNu#Q_AU%>;4pTiNlk~)m>t&Z7nyiEoy zYqBfYsl#}A-$B-BL2yuTn=|mSURcF1dJrN*zGB0HwLwn!0OaKd4w~_y&Rwc6kloJi z4`##;)aA7#;ew`2n<^l4J-xJ$A_@sKQ#uetkyBaFR;grv##LB(w02}DHu63&bl%z5 zg+?pfAc-f*RlGsk`sJZPxEn^S2wgBoxaOWryOVh&Wp}m@j70gU3d`lYSyVld1a0S_ zgM<)ojOEPA7T;FP2q#GJ0-9?3<2n7!LrW&pn?mojn8I_^qOvl#P)C4ydv8>P(O4p=?>P*nl$SRvlugWi`_2Po`x5H#U)=I z$e|fgV=d9_>_#DS3-cn3zgMbLKWS7s0Zu??t7E6)t5N*?8W+%duF9>%Eg|()QiT46 z^Q=sLqkJRTo`)|^hhpo6C?BI#KG&C>W+1NMR?Np84NzR(o!iwv?JF}qfI35+IePhF z2hDwkVGK+8vryKZEpMILkIet-vR|5(1qM<&yJY&6JI>k6yjIEoYD0P3JouSM8Bf&) zHQrIx)V1d5@pQsMwWG9n2!tkC>jyaiJxr;26vi~HrtZXPgR2ZWc$1}W`BGq{TGjqp zGT+)uAS>SSkFVO|WiPWU*vJCIgio8iC3z&jhOJ7K;~6=Nng?HJ?KVQJYDBgb8oEU~ zGe!{GZa;7&$)b*w!-L%FZ6ih18Cm^1r@id(r%LyjiXn}6@hU!k4Zo|E zOyeZiM_|fU2aS%-QdheDgHWmpWh3-_#Ma-FrJhCGZ3?XrcDJieh!Y#-PA7Gr9wdE3 z%JpojVKhZ3Qov4u_VTXZ*=gk*%2N@|CVoZ((T5(UL}|UuFns(XwX%k`W~3@|E6}Qo z)K%rf@#fH1+d5O9{ALxr1aDQ4&x$JeH=*jg-$57h&?Qx4ok3`eZEs2@)hxT3#BydtSgk!dlcr3s3DFot8N~VebBEGI z(R>N8|D-eW1IXuaA(#IFMX|K-3_4eZqy_&zXB2YoQD z>LoS}E0cWp8=f|Ey%3$qy|^rJ#y{BXAW1P){Bi7biB`Fm-qESPF^c2dLN%$hSc~)% zWg!r!q`+*DD1<;%Gk&UW7v-vdNsQ8`AwAFZK1}+|DI;NGqL(b~w*Na32fA|4DAwqd zvg=OkG0w2Sqc&lwxooeKASKgc81epg-T`GyN?HTG&H(L7+LkICp7NQKJo?o?u};&A zg9Vcyd@r{^5G4n{jG!hU7gNnf;;GIu=P3j0moIjE%L2*5L*ChdIzN$tuBj&peUutQ zaZ%tL`{k>q5+p+-dHfPs$#rt^+@2dovY&4vg&E^FN;&X{wpu7>RFUeoA)rFX7<^&c zCzm9+N$Pc`nd%Nw-{IA?7900TvnEZLpUBJSjEM%J8P25utpe@xp4duYsjT zL=)%9cmE7ofqE2r__nSa@p>ey;Lk+WhT>*%(D2KnlDyCCtvv}vXS~(wHa{M%HMAn$ z>ztE9|5&xtvDNM_Z3J^hrN912Fv2~jU0L8Orz7{g2oZGHzR$1KI}4-NA*e)==QlNL z^tmfk79a1h;aRg2K)WV&GX4phC>kbHBD*zyFf{1a@~c}gk#;vm|GA&NCOcL~IxTE~ zrrv7MMS*cEtfM64u6sov=#VJ#j(*UVEA#u*S0^7Yk=I<8CaAK5-mW79yoOXkY0`VnC^P7)QdJtMud43`mDBf`EFIaFa~FZMxf zQb8LleyN9~9Re)^~=@D(pscaz8L#fD8}*sEO1wZB)( zytf;XpRgaJyj40C<07#I)Ej+^-uK6dU#=pEdI+R(2@`Ebdb8aEvUk2W4toKGI;O(D z%@vapwqxCt6!GR{?OH^Ton*yUkTG@REI7dqf-W5a`tkYwVD z>~owJfhj|s!!b*0oBl&!DO=G*pMPj7F$j7`@T>O_>U`|ddO?FrqVnJ8o%*3k69Y?L z)%Agx^U9sZ8_aKFQ(W>ba}I}xB=*&MTh+l`m_vL{ztKCZ#0jrZl;Y`*QNFt6Ryc&wGNyY8V$G212ute(^GJEcD;~%}n z@VuY&DL&0uIGR#m&Ql5N$8-OX)!)-^SWk7HJ94JPkrE{{M7sGKfLwp)ph_W4o|~2= zc*soY?rL_<%kyyK1kKl$1oJ(^S86CLHb5j1k1}(h$lf!S@t5EaJ`8+3gtkqj>dtgB zv76uZQ+{@t*L$-o?v0Mh`Q^Q)o~DEv>CJijU5z`1#Sw>A%FXeaSa`MlM!HS}q9{*h z==?z;%=v_BsPJRVm&O`_)OQ5Q0cb{py`5{ZDv$UPR|7KQpN`7a=o$gV?-d!ICBI=0 zhRKHC(B}QsqxAZY(pb*4m;R0KRMR{wrp?LXZl1>XbdQ1=Wp(rpODnKjCx6(}AgZaWn!tXt}7b+ZFqc)xonH^-kfkw3zeK@3hcx zn!fLFGEBB=5uJ_MkBE}{lx=X2B8FDqBGoj13D@hGEg7McAAIpW>$KhZ?!OCE9Z;CA zSn^CZIzMZ`b4N(H!|hew&yL))bVZ_xSmC8ioEZN)(gNqAcq{M_x~8AatL|O2pONcX z+iR3^LbQeQb+&fx*!59o?YY>-yONhZ^71OL@z&CP;=g7YgUcgqE^D@rx5)YH6!Hrt z>`t3@5`3vbK)2di(~o)D)Ytj~L^O^ZYORcgs@g42-xtb=&%45?zj*C5?&!j1r=q-U7T&PuU&G_Vl8Ryt|M}g!_+wsQ81`p3(L(urPPlS4e9t9LxPl zn855=r69i(Qvaeem;E>1@V%8N-jc%VkZYuNKz_jU9cZJ%l8*+52xv4ntcCZ4xPBHk zFK8}h{=X~Fn%ai;ONVB2c1mX2Bf>LDh=HZ@?W|Y%x99S75O1=|qWimJ>sK+&@&`LD zn@!%=q6^cmmem4qtWjpzLz43aevh;qM#Z;*8TPr}h8*LU%M=aikNt5PTZFGa0rcA> z7XXuSd@V61yEZ~V{;{@6eb!^s(MvBDt_h7(h0$P0ep}_ZoWQ@Pj)BaMeBy4Caak7Q zSS7pZ?i(36*RmamA>T;-H?aEomv1gq;KEY?{#pR+0vXRZno4c5U|=Z4zGU z++5za+MWEcJ=sO=toObB3>3e0sJl~{tP9lsCglV5j$6i^r@ZwF<`e*mra{ZKbFCFr zcM~*Eo6%+VZI054$!ewL8P1RR+bmKDmKf|6Tjx_KLz}Z!n9Jia@e53c1wwvEF=1Ie7#oSrE?N~Gy2l<=)hcqdz z_ZX5*UYGaEzbKd0L>gh#p*6L7&s;0==jXIPnKnPJUV0n^lmRtW{Nio0I-*Q_&`dwc?ru~h3%}=#h4#4;3&4Fb0k6cdUz4ssX zBhY;T*KD@Kue+=*Hz`YFnGzz&prQ#4m@`qc5PW&0$#LFIyjDfRsRlx`YTt;;^c8&V z`8d$hJ7)d61R7Rczwc)jv3bq&+5XJ5|EmmPzqGksnKF?0NsST@v=s zS!?a5LHqcBh5+EBxCv=(`qf{tnoAd-;3)UiD9EZXty8lk1J%EE*0*^YR3Kk-nm^X3 z(TFc@2r~EBN7NGlgHcn^jUn9YQ?W~hEX(6>K?sdhz6Th=G?OZ($ILa(ssV1!| zIp)(}V68+V(Uo*O!`1{X3G*7Z0Lc4~r>$2D;>LT6fr0o+#H*?v@3MEQu`NE{d{{wd z7;ENw4IbG3#NMpmpGy+L4)ZuorrC`J(se+W|5Q)+T${>(h$r&d{q^Fitg8wU_<(of~ z`-5sGAv89IquD2vu2Pzv*&J77C;dLPf6M~&4g>BJ*47JOpIR7kl(RSYw*9MpQXd6`nMOnY#&r+1AMhj)36}D*9|HS z{A=zC_dXv}yHtZ5-n~=hSHuBxi1icaik}=o{5KkYK)KhSgYXS`hd#|R(nn`}=3DB! z^R0J8uOT8iKGiGzcO`2N4g}V;IZAk6FYwg`vN(zubdw&eg=Os2sR|HtE&WWZY;bKmzeki1#nX z5;FJW^v=g@(`?i~P$N~XVl~-Bquk`@Kz7>`^?)gi&6eF3XyW)Zv=8JD{X+;4oFg8J z*b)(46v803(={yD{v3cOa_0L8mS62>Rum}q7hVZIj(g#BrU2Zgfu8W87Ic}rL5)Sn zx?mYJ#dnwXN-~|O7xD_9&X8$DDw?CBA3dgk>JfMt$7=;+`E9|FOwQJ?Vt9D17qAi*9>gB>30eY70XGR*2a&a>s03p^GaPwI5u`i0MQ>Fkn~SIT_wp>7<$WRt z&KLCSQ8nNip?x)x!~}oOb|86*ER=Lll`<(zJQ@Sq6Kg+e_`s{^&?r?Kcg)6M+cA;p4eBh=cHktwUU)3?P($S zCOnuGgc6fijI@66O~WvFXR4WDS6Lfe!nMEnclJRPIEfU;Y;jL2@$-v2$*(ZZf(8mB z+HlreUIGgWXYS6%SY7?i`ptEgx3icmoi#!yHu@Beg*pW~laL?s;rn*MI}`cw2Y%x3 z(nRD702U9>NW3+sWDm6=a-upd;E$USuBcv!XZ%p$vQ znc_rzR6ka4hirDx+i~}k19TOcTVt%*u`>5HTzD;)pTdiJL!!}%kA%%8YX?8vlI~LkHE8H>O?UV1KIg~AbYEm#1Le$@j z^{tFck103ZXILIRXKfYV(<{yXWZ-<}TaX^n2pu7um1;Kfck)2pu^Bc4ag6s&6ps~W zK%O6$Mc8csM;=r0A54zfROtL8FPP;+lc#ZA=$yaQvm%F7ha&hUcq<}j5}G#8mL0#| zS4TN4-gGi%ck6{=x1(F;sD@vH_c%SpfBMYp?5XKMK5h(fRi5v#3@jx+&q$}EFm|JN3eL}PWNRzYg66KFb<{L#m zMdw)~$fgSPg0Yg=d4-@%AXfx%k7*12B@y7`csM6ku^Hew5BKV=N-iN@1#*({$cxQu z6Tn6$1xG7B1YwfyiFwzm1#6>ju^Y?5q~ zmHjkZ`===h94O0uC|)C;nh@JU9Q-c%s=ShoU)P^VFV9fQg#MdPQU2w$BK`7t64&Wgk>sZoXQeg zP8082#NAH#sSw~Rg1t$|5np~3%_}+fVi<&dhWn3xdeaGN^O~OOax;i*T{v3}V-_3Kn0n>fI={! zBMwisY3)>$doL{Rb0}JgIW;`6EqpU_Lu)yLBfY`U5BlJjZ3St8v`uH&_Vf#sA1<8M zL->}2P3(MP6?;Lc_uTLrdMbGdZ;H`+z~Hk9t!b3 z>-ysOYbQD5eC}U3>aF2iE)^r01kWN}obX)U2jEO-f|bD}uWqe5)>=&p$G7273LJdd zHEl27{OKWa4EQ($lms<~5r|WnGj0;~p1kwcHEvb;M_pFEau~tsk+_w5Q^i;`BT2)m zO+B4lf{#_Xz{ApRi6?rR80$`p@5epJKA!FU>uUwE9B<#XW#TdU6aq79M0VaCPw2a~ zu7>0vo;{H@M4GISRXj=a_@5bbf%6D`#bq1Y00pMf5dH_TpEBzCFO^$RGSS#0Xip(2 ziy-Sm1zN2}Wv_{y9QmG~JuIq&&~>>*NH**+1WdlipV?6v!TqcwTB)h$9dgy|8g1CAbPWP2PI)lqorN5SG{Jx#w*LC=ccic}zGbUZr`%zJhjNKsCC5biEM}r3n(NheWd8~DWLn88B2h2b?XmlV17AA5*|-DYOVaN zW5z|-5IJa{r;$1P)9YEZ^$;fUBHLAR6Lwb9Zj-@IlYaW=U6B@@T)xHocgAn8w(YKj zX53k3&kdw~LFe|bJP7EX`voSw&W*wdx5TNd=g1hxm2e9U{?Kj^aa@bD(PT7Nn2ZZu zB$cZcrREEIkZ(?<-^5+;+s7nHN}D5y5h#1nv9ZqTc>Dw5|p zAC>9pBa(336P2N3=sGBnyv$~-Vbmp2qX&4IhW|#<{ryDQ#XK}HXaa%?Av}L zGp_qQHP} z7G@T*=jsBzma$f;e&ybo7m&$1HK~cZXC-;1sa&BxCk0#5d(@1T%;)uf@5p<7FdwAi zhmJ@m&lvH(t<>pGp>WI*`bRQXdXFk&+AW z*`!#gi$qZu#^HDM6>^49M2Ye&zKASY&bVIC+qlSZevHnM|7KjS7PUYOn z);QugRgMJ`OL?dIDjo7-q?BbG_jhf;4+UhbdHY63lyIVykcBnI@9{l|c{VDP!F`*r zKagrF!tHCY3W-k?IZx%A4nZsM`RccPY=*IloLfD78l&+(a~njp1}0OH=>qnhqhh6; z5E=FSyyA81Uz+(?Bk6o%v#4(B$3Y2C|DAJH`YdWWE>!)gJ{Iw}y~DVa;ddSMJgsiZ zJ^%VHTG0Dom+%M+BQYUwmX~DR8e| z{H%(#%52;;$&jISH0loAV*i&q@cy$*mPo2QAr94|x4FKa0NuzYlFe!cQC$DxPQ!i2svPnF# zzCSZF+L zXC@sy&z>mT?zyAAUsY_!tFH2QCFWyEMGGb!yPcCCbI;xbIDRwx(I$C=^CkDi_Ot9e z(9+L{Fy;u&p8p%4_;XNXeU+F^pvrs%nu3yWF_sjuUXF(fOJC-e*f(U6~Ol?zbF{d{hx| zj=|yF!}^iN0d_MPcw2;_h>`3b81C`uwv79S_Kv05Pstal7wH2kkx3F1DUWL0dfyIj z(SE=$MnfVUU_( zCK+^fEw5cY@NLs4e>j*k6MfY5Fy(PHK4pqrlLqJR3D&@)gYcla6k((ut`{ZMpjzF4 zG5vdiO%oOECTW$7^63u&tJh=8eHjTe!oNO__KT`dH7t@Ad$Y2Vo^c9Y$t!KqMYaSh z6e>$Ah`+a80ckBJjR<@x_!zb`UIkrDJS$mQP+6=q47(at(^?c^A3(hP^iC|?1J#Uc zXCt}zkU}!n-K&$lET7K|jx3|Ogshk`^GHIfEJ65T-HA7KNspZ+hl_mqugo$;CU?^v z{@8@T&tc(~M}?ZDs)9(JH^Y*nrdU$O6S(!U@BVqQpR(&FJgXG6gK;p#YV9)3olFFx zn&jgLK8TcH4C=qKn^9Td&J?J9HoBF0%+IEGg?(j3ney*6un8aRcqBCXa+{mY>@xFb zs@9sEG{H7U4I&g|2>oc#q+CK$ByCeEW%c_l9CAvZPTf5tJ$Qt60GID)Xyyl}Or7_) zGDCmE^O+d}8dKO?!w)(K6TNNo+CpQUm{}pQ!puEB#eDH|3oret{X{;u?o3tKqkln+ zMFmcq7nS;tXQ=P=e|Bj_l7_)o0b|>4oGm(B$8GStaIT;2^)D7@;-y1NEWJ(0n&hT< ztk_GT62^Lm3KPdO$yUG_9I)2AklQVo<0^8k2{ih#F^PD}Z6FqdrA;OV+Z{%WXqG@B z#;y!ymWjPM>sNn*iuFQjmKbBWg@;jO?i6c9&D7w~PKppe#}_At3i7f^ovaLnTag0^!p8ciDAQjb9ty^I)&XagS<7_S(l&W`LJK|-sEdWD zLFYH5=GP}eB|);^L!)X*ZvK3LoC`lwrR#B+yLF`156Cai4Y{5;OBL(cgmbKT@EqjM z))zz8eUfn*uOHe=O)6EfuF7sW8R34+PsCi9B)Y5{7dQbAfqeg>3_gc=%ev`47M$RA zl0Pp6V;nIDyBAURjLjLtTZ?4X$wx!tK_|U?MHps^!)lz*`vUenIdSDTD1H6RiCkY= zGSjQkvUBWu4n?PPopn+se+A^p(w5_+WU;JZL{&=Df18UZh1)fqH*nH{op2xOJ@GeS zSs1O=+sLkmTULy&a^R2DMGUA%d*N!oq$BogvOaMtGLV?JTU#P@hNzHe16WEyfc+0b zl+4-NUY|szARsm|JDTEkq8WQ@vL>qZDwZ2`rX3<*!nK%o)Gbt)CkW}cYf5@O+q)}# zG&HBf>1!7akGvT|g&p}ymv@7xZ3~RSTTD7iCr{^kSi!RFv5leYL zrL{WVVxsR1t|85dw%K&3*HO(VH^<)V^VH*k=V5HYJ#@wFTMw^9t}NLRy4y#(;cy8G z6?s5MOZYfdrd(_5dYC+N@0f}m-3s&g6DUfCXSO|dUW~Q*-y0!Erdbh1k;2iWh<5Mo z4nKOgu9%4kDSuOPU|9P!7EY56p?=C#S6|cpN@>*c{juO=&!(whh#Yl8xXW(!)LVQT z#bt3%TOySXI)BLgkzKsqpBf7#Rl!Z)=A&Q~^3wy{`Lm7`QPWsMs|)!gN*w1nx$EYXDP@++S1sYyqn&<#F!vR_sMm~U1M8Q`(mt;Z$F7`MKg;g z@Mc>M%Kn>KZ!3r}y*DUAO?&qQR~%x_R{VHpSHnED;y9#cUB4G5*W{Xq2Xk zU^a+aM1kcYvDpj1u^u9`VBu3sX(D5RL5dT!kd0mkRD2(AZLhe4zXbyq~xVFGm1ADr@YT_yk7UW;~i&V|2*->yx1n}X6hOu z{{hkd_2OVb0u#i+>4O#0u^9F&VI*%D@o6uRN+9%2*OK$^byhRp2wwGG!|=SUyW#)) z$!BS30T`)YW z5UlSi#95{DqP~Ibkdm{FlxV5uVhAIT9ZdsMC^!nMDIOfFIr9Cg^EWAe!T&)u>o6U` zj&X)cVV!atp?fqjwv~aH-9(9!QB=4sW~|V%D;uXc%(&th!-hDyLc5Y-=s_N`r!d)u zC9K+!;`guINp5|1Q2$ieM+joT!mc$zSL6|HAXS*8Zvwt;p%G)JBCvI~vg`LL)AUY% zy2|WgY0MUzh8+$aZBtX_Vcj_N>Y8)L$iQ?W)BXFWSG&<5gXc*1-r5TiU(#c~w8VTq> z^E;)*VmC|}vdB9P%4A)&bRdWI%A?jLONlOCz$E5=*4G9u8ou_mQ_ISj-s~cFb>}o` zl5tO5bktrqnc*mjYYE=yg|#az5nRrP6uh^-&C-|D$A|`i;1V zQhhORN@URF+hkQ=0nct`cmDWSEFg2OGxRl$O%qTdnkJcb6=h+w`bE-J0MjgRk;fTh z{jwhS_AbR8!zN4{vB`IaFK?Z;k$?E~Q3Cpk`FsX9NIkY>-g&jJMVX;MBPL9S{19TG zx-+7J{qv;Xw!i&UO<%x00hupkPkxG6g2aH~I)@+3v|+ZuuK zGNc|UY1pkB4EQ%Tj(-DPEo%)JCjmhK5D1P(tbPAg;Qdhb&1T2tMb59PrB z);~kzh$dPYtgj!ltGBkkJkqSBmV9qP1MrEjo)6T<6>dHkCYyt z6p2we9x}Hbt(5S!j`W2dGuS$)@Y_6l#mc-lR|fxG#G8sH?@kCKlqicBR#~H)8p6~1 zH*%{z+ANM-*PluU?mFtVFXEg2u&l{}wTk(sX%35G3ObJ|s#a4LWZwD`+PG2H_2NZ; ztvvEu{Zg;^XXQYI?G9?XTTJ7BbX)u|*n#13q>fY!P5FhHuzXNKczuQ4x#s`&;oI+>8N^iyt|2{mrI;(k64QH^o$ciso}7N)cT+;VG?@@ z)O$fy>j^!O2i^nMbq8k|9VHKi0OZ@g^;L&(Ow!HTB+5O^VUpPRPI7h| zRu9q=E~3SfN92l@7RE=?K~_DX{{z(V(JMI z;H?iYV1wBU_eqd_VT;73PMyK}4eB3Wf+eD(-c{&nB@o z?b1b4b=L#4#vf$=W@pV#X}$67)CGS>(WO?F1;36mSPPH`Th8MZpD%eURQ44-lU}gQ z=yg&pK6t@N9ho-K9HTgwV8lsOSN6++L2i%_#70XK1v8Yh)a!AGJ0>V)wO!7jh!kZ} zsZ$?ri1ax1X+L8ctd(+o3Z8*@GJ|qpd#vc3od`IyB@t1J0@0&3)If)P>lww@Xgt&* z=y7ClNrFt;p|pPQ%w0I+;|s=(JDSHTpikfU-JlO~R8rd0?0HtJVogNllO%udB; z`!C-9HN|!{0nw+%;^E^aMbZX9xOE?+1(3Vfr(i!5%su;EwzB4UoVKg#&N{)y0NBjZy!3 zu_vKWivBg^Iy`1M8W?NSj)d0!<4kg$U6>F-U24q8Nc6XuZ=)k`7Uw4gd2m|o$ zPh#1MvRWgr6I`Q+u?|zm?U*J0*OU;e@b=hIs{GN=HW124d!7S7QSGq4N4tJnXZ}F@sQQY(H zjhPXqR)GJUx8=#r0!O^GE~$(ntAj3mo^Q=#x0bCOp^_5TdGW=z)Kw!=+eD7=IW_y6 zB#v&H%xfi0>)DMo2^vS2hZnJ~=%gm&N=KD7heH4+2k1v1kM)X@|NU@?m z0LkdR--IKT(EeaH^+o)Hh`+&z1`WcE+Ng-GEO0|oUDm44YLH}G+3}7}vvW1RNfmUf zlr0uHJshCSBf6t^CXE=clm-T}tc>LvY_aJ4eE#7neEg%D@4dLPP++r}z{_mJXHPqefC- z_VdfMF*Rf5Fswvc25+56nGye<#KZFFn>6QyejA-6ei&Jj;!DP=cFxhet-ut*jC;RB z=8Pc8vA+A0s~3tO(0`K;GYedcK-^%9%Ko`j7NWRSy?2SRz}Uds!rgx{gDp<#=7CgZw< z`?=@Mbc{ZgXSVj42u1Q@T5KXnNI6FPj-W`5iC7G8(ktR2BjYnO`f zBsKx*_)G=;2hW5>;uY}7apUt6px`m zHe3;>&|IRn=(^~ByehasO0>0+`i_73o>LiT$BH?IJuJ1z5y|nXiTN(;6>chuwG;XH z^eg=5-kg`UC{g)0@5Z0R4nTSoq;gA?l*M3u+Drt|k{SufzqD4cM|~dg ziROLsDDJH7qN&7iki5pzRnnv?vVyUVfou8T_2R?uf>bEHr<(U?Co5aCFw5eABUu7{ zUPjq2al!^yC0~CD_X&}^lseD$hHqtAc)&5Uo{d?RRSFoi{qfrx$Xw_>FM@2r2^@b9 zgh}ZB?=fS3@37b~yT(g3f$DHxY}yL!6s>*4E;AbAsyBxhY_s^T1^Q8VqA3@xB}Bi) zIv5a%%&(~)YrwAOm`-tg9?*~dZ^fO@)W^cwsmQ*`^ITRNo-K+>~ z-$cUdW#7&>?bIhu7FLPi4pem~cS}D-VCp0u%1S+DKP=Q3Cp*F)aRUI6NU{`oL78{o zBT@CM0=7@r92@n(`=@IleLskLM3mI4`VrTh@>jdgxF^hxJei$wwe=N`g4IBvgbv|v z{B6)x3~6yAskeUCNl3Zsk>;Az&OVm^=${4oq?Vm!oBZ*yKlE+4QCk)@gfp=J!R}k4 zL6&#J0gPQKKr|Q!D0HUExen(I^u2AJWNI)oeO-;f=wn!oH4YA|$FSk3t{#e|+3bSA zTd9ryK6;dHP+HD#+zEv2SJ4BuVVV*m;O5O-s&W8xO@xctl-j^I;aL+;)fH)OrX0NPbx87*5)4z4>D{b<)FYTEQ>(nn zKc zc>*%>lYFt7Nv9B_G{SApM@~NmdrBiV|0up(m&$VkKZ+i7_>7Y1FWTPNTkA@Q#Lt$U za`k0Z67Vgym($2+v*}h)hI9dmKe`a-pEI*&_C$=%M$!f$fLSjcms%Ux7j1{xV z{RlQz0Iw!?FW2qEGjr(>%1h#2iHPq{fCq_sxA2KJ{!&o`)*0}6VOV)7KYBV7&7^$A z**UrqRkx?e(Zn!;Uh6wuXoF1iVI8$UW#TZ^zG8#5-0LT9ZeFCDzpiu!kn5{YZChsT z5C1cR6fEJ%-hFYeWOS5-OzgW#nf%Xp7dTdT0)izAdpzG2;P@K~6b45NL~W-SJ%%A4 zY6B0PH#j+ z*jtbj!qpf&UrAoS9o2vwSg@Yfzxf7*QLX4w$S0|pj7M<@3AOA&RObqnyAzVNWPGt^ z;sfMrYOINj74zV&TwmxR0sXco_ZV$_PXH0?NJYq0seCm#75Li|pDXHsP`($}3Na;Y zu+{R)pCXBSeo5PkISUD6W7BzZ{5FMso@+11ny@fT;SW;mH6?c0 zp5I+bIXkP&b1in;mbQG`pC{BH_touH|8`36J+GtMInjLlu9-9?y$ld%7%?3?&)f}p zTl9GO50z_0iG=i+z4 z^gxGKaUlM_cMyxW3^3-~Vp71#^%?(zp%1o_1#(|~+M~H6#Q?oM^p9*Oc4!`H5>62( zAGtHa5&*VpQ4D}V>i+?BD+nIMz6Q8!;ZO_gEC?H^B#J`#qP|okudbc3H!^%X(ucv^ zdAt!Y&p56GN_kmsg*&{pmo@wxHnrd8LLFf**EL8Q>YsIF@rXJ(Z?<>aJQN*_iyKGc zDu))ZcI7v5U-y;G4{Iu5RO&qfVK0QR_tOdBWl-He2}g^?yJ5Ce1cPs86Ci@nDu>nv zq@$EyCqT7KK3yMROF`!x0gi8}HUnTQL(o$9GEDy#ps4uGMz~m|v#DveMsOr#V0etf zhfqBPxkicGN7tS zm-Cdv>guQ>B1MBF?~BmtG@+J-pQ$se18{$WcZ=sNrC`>{AF;;d+z%*aet*Q)pjIZ@ ztmnwCTB2Xn4crDCEylPHN;tcemtujP%7r1~gc2nhsR91?O@Qmvxb$!{(jIy{iSVE} zXO@d78E8j|9v6QOAh#1;L^lLBA$kmRAk6^}4G3)cSlJAsDh57lVm08YUTHQ5*%Dd} zAD-5ti|z@od(M*oNr7M#cu%WwYOYQ}z*nQI_exM%#+@?-F}0FT|YA=Z~? zPJAr@I#3Kw-+U$=3Kri?H!w6A7%{|p)X2dYo`ZBg`kv)H255Y&IV_$hwccyFIiiF9 znMx_NYKX4PN0ExR<-ZmHlr@q%SCYr8-(BATgYy5pg2@NPIq06-50?ODf5rEF#U;=8 zCk^Sqy>!bqoA1r-&7Ucsdx>!*?44OF!0FEgG4C;&+^g4jW{Pvst}C=oHQ&>Wt?@@uzDFQI9@d;1?~_F#P;vMRNw~J;x4)xN9#i<=IPe<*OvBabl<-$G&DbwtCpSPe znZnQIK4pq%KX(L@1-X)>U6U?PvoTZ@WelFPI13i?|!@qCJFd-3oy$5?2@u} zFWq}CfcMnyA$}CQBgv?>CJKmWtLsANO^$Ik*5-EB0Mt|tXoueMwwJnl9Z<-1kS6@+ zI{Q@{Fc1YFJJw*fHuFIY(vUWPBeeT7PGPJfJhG4COpzG61Pa+(4sS?dWU~q<>WSpX z6+ck28S7|D<(#~R%OBEqORV2QX705ws!4W|LihFpj*m<3ET%3JkT7-~y%$Z}gnUhC za#s;eJT}b~n~}h7`$hF*!;RXR)kx9ItstZT5Xvr33w^b_Rq?fu!a04kd7wltQc@G| zjg0syA&_Nt`2JdVj+G6p{5Vcpz09AGi5j=WZsaXFE{VIvm`UmSrAG-z`edFg5Yo9K7dLpX+Ym@9}mxD8GmH9#R<6A@p<8~ zE1$VNtql8H{+NuUsM4luT;-S26h!tpj|J-rP%pnhh6F9#(^O_jW$huRrIIG`8}&kdvC>jw?X~UxEx8yD(f3p zNN>Y=MlO%d_CTT8qPmYcfL}^ExCP)lA=&ej-u4q}7fRVMTZJrh8-o4pm`>SD6pwjA zjZQqogF(1(P3)W}r-dZqct+rnL_i1BXQgh?*=C62nV0^qKL@$7KL1!2Bzkt&IrA|C zhFMQs6pEB`2LQ+)`z$pxEyw>wqmxq5Lhc*=v*Ak(zJtsW9yIhlPZs(dU4yD%1_-05-!zKHBfdf)o@#DNywp}u~t_ua4B$(ZAHN7D9_6I{#<%A3tfj06s+G<+_pZLzv2I?$zbhVN2niOGngO(B|T z8YH(wC8piJDk5s(kN(i-Q#-OUugkfi)RUorQ>g$@jg3hqDjmk8KjARNz7)}_m3zgW zgCsgRP*Tz5S^C)zAjmG;L8NN|`+R|2qMvxgRXgdZ?uZ?XAW~fbLtS;*4`Hgx^al8M zU*1`$4uZgr<)HsZ)>}qZ-ELpQ#HK?)kcLf3DXDY|64Ko%-O|#HAdPf)x0JLfND4?d z2uP=NJlEFyKIi=3cZ@yw;s$@Qt`&35wPs(2NpLl~D7#$P3-x`^jG^r@)+*}TI(H!MRPGgy|Oe;DAisH%>yTtp_eJx5n{pvivXoxCRiXu_pn9RKEv@1->H09 z#ieGc?x1oUcjo%WV_B#{dsJ&wSLCooB7Ks6&Cnj&uY>Q0j&$@;7q!}>6zFWe9eFSD63&Pq|+B@AC37|SJxqHrCBGL%~C>Y&F zem-E0IOea@Xb-+19B_h7?Pb@M?zV45 z1d=W8rO18~1a1M(y)Ot4)9s?kWr5YQqAm1oOz&O5@Au6UM7x{ER?N5Nkar_0emSYo0ut^^DHe|lnlu75ks9i>dFz=k8H>4^3X^6j`sRl?_8hlOWQ zBSRluj3zW}xkBct24vz~!(HLn7<3wR-pRiDCBl{Rfgvg_?<~p;dXHxfYA@>ndde0x6-aBI?CRVGGTuW?iz3N& zhTMo;fTc2BS@gZbxh@HHpAQgFEfA?pv^9u~QKc(>q%nu%7jqg&dDA(HvPxH^l_MOI zAf6@bD=qN&Hq_6?CCxo9pG?co7={@RQTa`+KD(80QqU>(c?rC%4~yU{Pc6uI?^^eV zBVDF#AxUx(f=f%V?Rp&pRe2>@P?@ghU+4-vq_!D$&uWE4pw941yG(0BeTj6iHFIxK z;nj1a$oVHTK)XaRq4Z7)J;c}cZQK*Ae-o(vwZxa>;jgEb9i`z(1phlhgZFg@aRJ6*~pPo6Qx;! z$*(VnpSb}kew2Y6rYH0Xc-+~l$L0u?Iz_QI4Xa&qVKT(6VTh)Lb6Y`~I>j2b2A@8c zAyEh%=c8`?BoBmW8a`2=L3`a6Z@YfzgO7(@+x@2P!_$@HRkQ(&D13hE;Pvzs!9s2)1|g8DISN{CyJ92@rA*)y0#XJTa|0A%`u% z%S`A=PU|p(7^Aq5u1YeCkDIMVeeT(N<<-$=|DLkGro~o^+3|Hj=o!)~&8-RNt$GSq z!0Bp1cn46AnqC}!p=UQ6mKYuqZWsIr z7spG4FNnbq7b~}*FnbMA(F$EV{1&brk8Z3`#>bCKrfv3(_e&1{9-L$%qFsJvgz4lI z-oUS4otC_H^Qe#ozmO?bs7EfNDYb;7Hp#~}HJyt0}y!V&TJixYEz6Gdo1isjyPMesv4RL6zK#qn>#3wyKwo|)!1b2~)6?8B4| znALpLF{&osndk?RZup7kP8N@in^2o0zr0*F;E8rflYWYg|Ayj6v=gf(McD@m+kcToi(DMI+31k^4^V!X&vY|?)8WY8%xVtU1!3JW`8%_!KL zM`is@bM=B*8WyXadu%NV6{fDE6VKytZ*vp*roiB*kD46dj7ACJ5$!meMh-*ANuvx= z<5U~as*@vM;4RTBw-k(T?Q#G5fEw0`f}k|Si)w}od81Dw@i@vNje7h60e_Rf6Vjd} zZD%tv6=An_dbR62#;TH~A^}NB_Vh4vTwm!Jd?Y#VuO(!< zem-NhVS;&jdT5|EcTag}cmeB#0s?BlX+zViRd-Vro=u3I6Kdi8MP(BcH57pwM+00y zhU0pk&zCf~l4+faM2Pee#YWw!d_aDkVU#4@bZu;$SU|q}`=YnFMlv3}BN4rNZw&xt=hf90OXyJI%+ScZhuQoOMerif19^K2p8sunE zoM}U;!uxqNt}X>ovhJ<8V**jUl0NQ*G--CbwWf7+!jCT?;L^`sn_HA2tc`ISLX zYiXrYYc-z?=H0#X!-7YPvni%KO@7<9|6#yX!kfzE+LXpbaGf~>_2Evh)7+m?7#*Qod z`@>EkzO*hw5r-3UdzYvIe+46!}A|OTeiBnxv@zfUM!-& zyZQCAji$8uB<|<42YPXLE>n?+Z;Qs+hmUyPmWBB}^}mOMgS)@SOmrIlcfHAO21+u4 z#B4*v1D7G{a9pGx5N@g)*y8KjZm=}MB>;5LH7lNDu?JIN+d1G1svVPUu* zB)+toDPo>X>2*x#jHh!tjSZ$BwoV~!I+s?qlqt9=OpTA|lnsU`g^8d;_*}MMYd6?f zUj5!l>Ymk>fW1E02TBM5TQ6Dbz`$f`yZ3b}=P(Zqp+ur8F;F9UKLZeJxhx@XnbHL- zn|;*7P2~#hk8d=xW$^qv&~AW&kr_~Jd=6CY;x7TjV6-uqVgf6dk6`Wr9#<+>L`XLe z=&CA=w|Tk=GQX-Bsn;IDBVsoW@&Jj>5U5bh!OGFL)C$?SHddXMA6_SWjT;^Jw^f(7 z=Sy;JZ-2`MB4fps>$TMZmW$*GpkOCw`QFVu(XQ{C2cu_|2tdTJ6zEK-VGhJ2`Hn}E z@s8~R)rW8DCC8Uw-valIx2*Hl*cV-}+J$(p&ODl8Be+B*&(`4W7M)6m&#i^FLH|RZ zvuTqdurbVnYu18>)8s2=9av?RWFV?Str!UbT8PibYfL0R*S#E;7Ti71QeyYl&ERKV zd^N!`;ZbaL`N(OD()%=)cfV^^X-+Xg}hhC06i+DqE7|E(w+kEgvar z@l}dSpTg`Dir}HJ94&mu7D>wO8bE>;H+E22srM=M(Nk@ym;@m{TPg%<#V-&Fp9>>S z_d~@D;eq!63{ph6D45XMA&VlypmsZfM-x%Uvdd6u! z?7SKebibaK+JWy3f$~tc(QYBvW8%{E_~*Amw}VByB(F>$%8VAevZw>FP8v|V$Pv-WwuT;N{o|6c{FRW8 z{aiyuim-2c*(@mRY}_qns1Tk@K=G$+s=JUkz|yaZl2ZIZA7(x``P66MpvT>}Q5^>- z&z&jyr!v3YzkRCP^nKv~S}nh>|MXrbpJWmrxM8bsFZlf=;e;lKSdN##wEkEiwW1&- z@u`j(D9RW#l+ttd{@n{R8<#odRP*th?6@+hp00C*CVR%1yQ3?pBNC7z2 z;KT3)p()@nT{a-Ud%ID&;%PuW*abe=jJqBPeRpaTs&k%vzT_*s&O$}botvME-SZPQ z7L|-O6q7ivUZ!a+hNZX2876bBHyf{8IP|&1wm*Tf5*s!O?XmifULD6Sw$gsN)88(` z=yM1!mFAcXy_R3HkkBud(0tnJyG-xHE|fH2g}riEltQ{AkD0y}6sXBU@m1ciXjdDG z5**YVMku)Mr*qq9c5O|gjM!YC9~{u1Y^!YMv~EWQln$a$dM)1#8+mniC~7ETDVm;y zfON-WoZ!+R7A77@=GcQfI!AeFU^1cjkC#Nxf@?T{Z7r))-N?zI7DUM4bKyWCbMT|M zDhQRc2F9k!?mwC*nFfSlXZM0TYGAu@nHKZH3I9B=VswO|)aOot3U58Cn0}QG>OMRS zpq;1Ko+wM&SsiCN0IKjzZV!TZJ^>{>FYPbGj{V2X;Ppm*$QAV6(x&nBZd_;e|^!PXK`D4FP>t z?%qAh`wPiu59l8pg!j~ZuQMO9&?+_P_V?%*I^7x{f>n<6zlmu?zPa;)$%7o5B8S+Nb58pN?jD*{9%Y7DiPbq2-Go72$;o2Tffd`AvY;D#}RaW2FNAi4z&YF5sNkR$Lsf?j?xAD_AAlzm#GOf zOuL#9LYV>@vViG*u#G!@qpJ{tO-_QAIeiE;b6iT6yr}>zm~~V+kiyZE!L%p|>ZVNb zXTC2sE`fLTTDyzgK&Bc4`vXj*T;gS5vbBQu(x!S9GzcGXp}{mIG+?in&Pj^PFYjK8 zXr7Yb)i2@w(;!4*ZAL0bzup`M-eRKd1~kD~Uxsb}C^6X>&d|x*@8x?Sv5>>c3-ybj zD{=9?+M?&Qo-8LzD5jSA0Ia%T_Jp{e)D8Y_dfZPR`LO;SDmdBT^IB>d^nD!=r!dtA zs2IJ&U{T5wF}6{i0OlTI9zY!0t!L{kRX@PAQtK^7sZ%EjVJ9EQzniKW5OrJdrGV#@ z&-(C24y23yJKyNDo{`#Ai$>-Vahknv23ml+f)4Lsj!X;S*XqSn2Oalz7Zn~#D65p> zBF~ANbXE_=54nr(^_4QD))74sG$NJqUaHlZA=YJs%a(qreMvf&N2UJq2sr^?@*gv8yoKDpYS zdm+N7%dwv3IB%~6?Bf9huV(}4m1>B z=pge8bPmcet|$;Ng|%q1a+1U0Jt0B_NG!bd2Ve!8b6M;@R*i_uSC=V0bBCAIM24F_ z&lXTKa;CxX__+(>(N<#o@D--li7w>mfmpWoN2`K5kFPtcOo7#KlI&4R^Jd-4H_@B~ zN8pr^D7%N0AbocEd;k-CoiW|a7-IRfZ{^yM)mmH)WG6$)5`LVgZZCQHzt4LV#f3%k zOzL{NGK{=i6cY3S$?|K%{F_&GAU1wla|CNh-nfV+sDAK|z+r?Edf(Jsj`CGO``6B4 zLZ!_oGHM4M)UHRao2&E0tnmr?JdelihS8eCN(CNJiFK2QCRE+LNG6U>vupVlV=Aen z3t9BoB!?weXmR0XI@bBm@n|Pd&${Os19NV2^5NUMcZDak|2}3mG{dU-QZ?rDPBm=l zt#hgXS_+&{`;bC|@P{W{^gY`CO#W%@#lVdtPMGp4rV$r@!4U zc^Qx|i)Qs{o7NAC=J>_L6^DlsB3;sOyAn$28ffHmd*m9^rnWQbQ~OfLG3U^77Mt%E z)XgBNm>RPMoc-U!ANyF%4?VuVCuZhZVO!SuU`!gM%RTQ9SoRgxZh`BXVw_M&&vpns z_8)j?r_hWYBnUiJ5lql%2%H{rf{s{%F{*k$kKDdQs$3ef!i$W<2X2+GYd{y06xj+b za}N5m_=&p^I}lf_za}0Sc|1uv{8c5nLy=Gu29=q3Sn!+~b&^Pj?M2G3rn2yp@3|5Yx}{Sf$ba{; z`C>xw1>|)V*DK+01iv^cyYhq9b$^52>LUwg)jcmL6vUcSG5;U_+!PmxF9Pxrw%F8E z{=Rnx)I$`y2g&Y;r}g9V%;8*W<}~g^Z!T6sU&W|qeE-P$=k@+ghj0;eY-)meL1QU$ zzUNi`XN=Yq{g@zTr0ys&G>GrwyAZy$v1$NIOeDgvafyM%F4bH7tYQB61UvPxRMZ5cvF?aGpwb5sSM7lJ`Vctz1 zT*!dq9Tk?ofYAMo(+N-^OwlkO;kvK6o(9URLoi`tvNU2Ib0|6cBT*f*q4X3dN7AiP zWNb2vOy9dp_usj<-9r0p!E0z+Ck|;dcAxeJfn-;P1HBQI>*Lit?nU>{?JH#6Z18dH8EnTseE@mgR12el>|c^l zlLRao{Ce~C=<@+o_ERv!`o}4KPC?JJ_hZT4V1s|*i%{%X9n6k*Ftr3S*J-25dGUKt zpDrwLSKH=x5~HqAj?}KkV)Oe9yk8&9LiNNsa@1wfX&8jQKk=W2^Vcb5pUWi0;rC$I z&)sv-Sv^lF#^T)L&6A9#P(mLTpW}Rh9R`(hNx{f55xlMp@#mq!~v4r=)=j?N6^8&S#`!IrgF zaV-0fF_nu*X>2G@-{0D{*la1+wGm*tccGR1kSa-hg2T@)C41GVQ?5wo9I+bw76%u#aOplw1}6w zlHz?+6QW(1Y8VL3yraqzrWdBCy)){ae$})eI*mf{_3C?3R(W(=94f&!d6)TJ){4Gm zzMa9qU#RTxw_CxoVZVOuQ2JbSHM3A{uhRY2mEgKf!=OO@D#a|3?sK&zzZ-CZ(VgV( z;B}ZrbrH!`v&vySp{C(tOr~si)QV;(@{~;I03D9jNnRHU1Aj zegCtp(x-o`;ABYCO7C%s@Xz&_1RqM69{9e0Dk$RW#vlDt-~Flb2p(E|&-Pae4-Y4{ zw7kO+kv#?#Ge7Q2IeyFn89L`eL(;;NuC{xSaybQrsLpl0ZK~*`^nXQ^zk*J#=AIK$ zum6o&HtiAiB+d7(HY9k2C;`WF+A;VVU*q9)UKWXjrCy*q&E&mt56TqMu|CgDYW?)? zNE!|Q&cLemCz zt33YA0LJs_D4`c*XH~wp%u4KxLxH16gu2Uoc`<`jH#V>sR@^-z-n5GqrUj*SFZ6kEALn5xbGFPp^?K43-jxZsC|Pm^P;E!V1s|4rnrZPEQoKUWQL_)9Y?@XLy#pLgBc;B zc=95x>4BXgB2b|5$(rygPgNzzk>q%vaqEY_s4j$x=ha2F0@3#~e-o^&+BhTxyr9QD z@F--4Qk0U*sLsHde>%ul4L9fgWjRjk5gO%Ik=AR)bxU*vI23#uX7efD(9W&dk7cg} zC0ic#VC=g2k&_;FX~OSwG>5ZcQJ8XAMAzY1zV~7yied{7ixA>bZ~cVsMd^Fv*fE2i zg<6Z_b9u

oC$KK~M3+-$~F`o*O1~&LBr>_d-_VQCNkA0TB;ujUIm|h?*iuET=PQ4`;`nN?zAJ1i$3u8(~2(aTh!Hp9^KcQ1)E!E*dS zcz*Orf}!_$*a>A<%@_NGQ}=fWn~V_o>Ju`&3XPdo-Rf7a`={51OBc6PEn=;>Uwhs? z91t7`e(r*fB$%$V-+pb^NL*=}^Fo)aS5GrqcKAiU&5LJ*gt-*}r;sNabYb2{CEQn8 ziU@?;>;VYy=jPv+JNAzCvK7U42zaz=+(!tDl!OERXVD~BzX%-YOSk`e86-(KQ9Ksm zo1Bq3#_&^AfL>CtR{V8Qe^I6w5_sf|hQ+tm=P?vTNWrK-`Qcfm#AMf0o~^&gAoeql z*#XF(^VU~~`en-DC;C8w2Dhl*dWwazbhigJCoJqBOs<$4@fPzZNA_JD0e0}}1Evu} zmcY;Y5S&8a7x?}sSXu7^Ohd`uNYP?;Mj~op0{|&A4DteVnZ^PY@d(k53>Cf`-mEE3 z{k*NcHSBT;Uw&xyXyni zO|Fmea@Rov4{!?w(HqZ~6!^CCrK)@EjX(}Q{4RLCDgIC==fRpk9A;72*GSS{Lb{BX zexe=_Y}tp8Wu;mw4g7GBH3%qC18}%(Kz`$+-mEZYxkCFJB>XLxLeF&>q;eOVUVf2~ zM2*7{bs&2ju2bQj`_a~|Ay)2wBOIlJP&aP1>%ev}nO*)_5YhKf6EHMlUyE$B__a4I z%iul2DkrX||5X*^q>KNV7cLgN7qNp9!q9|zbTfidQp=Fm7m~rsdSf$Lu9c7)yCbUm z^HJqa2TeEnLK6=C8(F%D!`83O0Zt>;gNolpo~m&`Fx;u%16R?f}k8i0pN%{MT--gp_zEnKTaOQ;#I zruR=)UF?WpFdJ%UyB8=7pq?Fcic~V&rC+hxe4IYBGJo>(f4Ud!@W|m(Y5@@?5b1Zs zmUce4BKSuz;6NkI3x_uL4mAlUh(0Wq2?JROSC979XSJELW?doAGaHb`_7+Q}3yDq9 zbih$PK>WeedZB9m{W}Az{6(6NjVz=QwO2)a)3`Ts3o`q;-#;lVX z;i#99B~(+_LbqWD`* zbruVPe<XC1F;k#g;M4|-`cn6wY~4#$ZK~kD@s?~o16X02;Mp|I{GfLz zLASK7G0$1@S3%rfD68TnP-{%@>)~gwP?QQFh9RF1vmJf<67>RB7aVt@t$pT((zsb0 zNo+-K!cYeyXkNH+L|xNod5vXs^xG22I_w9g5T11D6kTgp zHOF2ug-5_mA&i~I4%g*g$FEv_)K(z?je=72QVH$H+$8h$zpKwG(f_T<{I{t(DGyqa z`diT;?0%(Jz!;q!+)>W8CDFJHcztrO$ zXBK{HCiF$6DTZtO%m(r3W#5ApGHmB9NZ`fd%*M1y^mEi#t4BZznx~OCYw`X1dzWRn zI0hWsW!1oGdgsui9c6P}$GGOrY6=8I)>e^c!8lJbbvto_scsAnQ0G<=$RH;9fu#%% ze3dNXm^fefl zR*{n7Nbho(MGIUa%B?!o4fmW0K*E@}%15FdqCj9cpcSV)86z{!eBD(e-ogo*GrM?u zq0TO0%VFZ1ED(!&tX{|Tv@pe(gnss}OVpK2;YO&y7jG;jME~z)rCE*};d;4FMb2yIeM3=kD#NVOCwC=D`Z!$NDQJ({T%=$9G;vB zV=da!|2h#UH)NNT-Slj)nB$s_fg34Nlkp7RD8>6w$GAe>>2J5~Z#q<(>PKG8QpbXC zdRB2#y7-}=&_5lg+P*1iMs$>eADSM2T+7&l9gsA0sm-r_f}a4 z;$k=LPvA&fvJhMFj0yLT?u?=~*av~x642#6}!d;y9b;lruU_V8&Vb>{0&avnZ^JdpYY!AA@V zF57AJ-V}l|i5*SIUG%-ifHVt1W}7zy6(TgL=19SInJ#3Eb<~L* z%y3Qv<)|g^y|s=Mh|eDxy&hnBGT_^|T3Re0|5)z0Zh^kz)6>TJv8S8kDCJ&r6*zMv z@)xiMa?*9hUejl6)0;aR?JvccY04Zc4bAB(%&zZ-GmHeA8{TrMFK;~33N5jq=on(% zt<&2P-`FR9Aat=D#y;Z7i=xGDp_zZ3Ipj#l`Xv5DvGu*fY>UDNtr~q|ldtW2cXb9G zKBnr3mPzM1sm71kQc3zPj3l(LHxTk(=I=!h-@_c_C zncJ46iX#XwXS(~rTWW|XEgo`UJ6A;-bD(WGvv?Cv(b7yu)c{?RuCnEf!vhJ4oG?a~ z1d(aO1>1fbrX~*LrdyxYEG|05dIN}pzEtPi$*1(%Uy~FSJK<;m@!U7T#CklX`SK<- z`Af_1ggI_80_xqOx$>qbgt*vOD!x1NDrewpMY_&jhZIMM-P}>JAd)6Nd=W%uX~Lkt zMkGdr7v9z_Bb&kZk?d8!@jPt`!#BPSam=mzf{q(Jg3!ak!ocuEVW4knmm+tv4ykI- zJk_IKNpg{As@vBdJu4rnSwpgY2a#?B=r8fT*Nhi+7#fmI{Q|3GPjW;Z}SP{?|T*uQ=#`zrk>ogw|uz=^x2k%Xd+3ret6lm9(^?fd7%%- z+Q_%Wxp*##A%Ke7l^1svSkpXxP*RVoe5I+8n!nz0xsk?V`*5onb=K?)HH#qOIE*Jy z^4aHlkNa^v^45>?7;39=*vM%4=QvoiXPyg`+ZqXY-_SELOFntUybQgg2bqs4=uz&s zhi;`Mk7kP^CT&NaiW$XZ#wEP!0i1W70*S^|2zQ2UI0gT*ZF$|xgw=PqijP})r{Wf2 z8@}0Hb|y`gx((;M`FQAb9}U-2Lz9fZ6~&{^nW0P#0c?8tOfH-?(WBjnctqgjy^&tl zU1ZB>hCRCVtDJX?2b=AoxN4_<~=eEBa;g}`x>d5wFtGB-*W0mPxbL8_i{`NguHCtS4%NH;7JE9lJDFGX>eZxql~D^VsFDzdjL z`!@)IZ#ym%K5pSs$KAivq;jkLU(*JaA^GN`Gyew?qw*Ra{e>a^;^zvTEo#Z? zuAQ!apO>KZ&|#-Ng_GaH&;7RuTs+<^3k%1N4)bEX<_4CwZ@Y_7No1zI}; z*WgyPYfr=&*jBEhpap=eAiry_Qjr1#g#?Y}c<-=zmi>xHmfv5X_@j5Rzljhu)Bb3e z;cy*p2Cy>yeJ~95x;#+j87VGVN6w*FE9fSqfYxOi=rnR`Kkz5pj8Fx&DCof(X-}KK z5h^cxeafHh#ro@Xplq8{$MxHAMtl1td_N%THbluHQw>uy{}Pwl3OqsC`~Vxs8Yw+> z0x&o+r5o9r(d)MqVqq^AJ}+y(9ydspMw#-%^>Ay;mO1jjT<)5>%BiwCvBPm;%(K!rR!54|4dNe;ES3 z(E9BeOWeI+ZZQwPpG$X7atlis{}+9IF6HO{Y0_13w9w8m@WXRWf<0}J7j%knRAPl; zSZT+I2Mb^Re8}AwY_0GMvwp@bE#+i{f26(v@%kGAE_N;)vKBF|`Et?$fCrE+L@r{( zO`;5hM>lt8>K3vca!peP*AB*C2VTKwL^YPv>G7jU>feJdV0{4BRa|jSA%`(hU4K&i zZW>k+5x=o1^kAa*J^(Kr!18e#?k;&fQ)^mD#QH-__uGk2TI~QKr8l)MA82Ae43BhI zAKi#~ksh|f`^g~fjCqzNjTP0wfuh@=;e!ZKa2Nw-n)dK?{cB`NrYMYbX&!?U-7j_8 z?Ox5I7Jiy5zw)A8#TfdR-B$_<4|&iBlo2v}BVYsO@lE2_~~ABaQ0br(uF4O8oD zu1~%23JJvXmc4g(qwFBt2s^vj42Jh1&T2|Kl!f$?Ug377MFTR?l0&PU73=oIB=6afS{q*s+i2-tl zTtn~_AO`OANTX1=8#)V65v@)CbkkEhywx9W#{9$lJl01yPh};`Nd~cT|cz^R5GiiCUtexzndqX#5#g@ z=O4Opz3#&?QJIq{BL~yx^0E=6YPHL)fz`NFI8&^{XenfFWUNPhKLvVtL)TLH@=0|! zX{QyEwC^fL+8_vymBdYgfRvyDKsQsF%ASTbu|6yo*GfFv51N~0&3}#Sxr|>Zj%`H- z&x2%IDVYvt_rA|BxB-dq|A*TeR{z4`CsGKfntzB-9diJ2*zW9t8bG8o3A=H6xG}Ri z6nBtld^~Cu7H8@A2oRTrhU>&0&pS$y26-YP-Z#B@{Sfmd%{J6lm&^Y!@R<@%F)hiH z;lf=99r4FJ+*G^aDA>GBnK5Wo_(g}ilsU*zEIlElb`}8b+j+ydv}(#8X%XCYBEA|Z zzEi{*-K8Kd&BL)wu|6*KWnftg%}%Hp6M~YEvJDl$mdIze0xjWgt4=f}uv&x-=le>+ z7k7+(UIL_U{4rxk8ye-#1^N$5=mF&guafv`q$Jis&f0)F&4i~&JInIHW9H1hP(Zna zbmuTF`*w?G_M)~#rw`LD7A9@Z>Y9bB`O8hYu!hB$S={_ah*#nIG^$*{=f-^;LcBh% zKg&$>xJHagu#gGn1O^5#;TjHsK0k~?+DnK_y%$Qpsw+;J-z|=_HPx$y)%Cg*S9pJQ z&h#yXw%G{f+V01M9mZZuCedq{m|xHqi^`1t&d&$%J)!yKUVucD2l%uq{1f>x08v$y zYEuhzS$|5v;s-;d-p$ob8h)(*H#KBb4qZZX52c-#-=3<&FwmJb``lI*L9N`31-{4KIm`@ z{E8|#O__!HBt8T+M{1$PB~Kwp6!VxyNa?k_->C#o40P5VxSDhx6cqaOTCpR^4NpzS zSMxOv&PKjuh^S1cJujNNzBdXu{aX0c=v1d&6LdeM)#_g3^l|lJESV*;8<-EfuTWfF zvdZ->!{F`33Q){?#MW*QBqDTc;zc~3-zD}v@d<}%==GyY4^#_=4qK4ZlO@^+f8h~U zaV@D8nGd^o=%_5PEN@2AWB<-Ll-$)pY*c)JBO__Z12qS~mpYfXGC0h|d$}}NB<@*i z=uDCoZ(^yk`_6gUNGZJ^^DG#mUHI78GWJB>yR1Gm6-EY-Q)KP*&J9@bNYc}#Fn%KY zhYs}z%AR=9+2JAQvkwCL$G5K9;+|cTnT?a{F{%BqFMvo%~y)Kc=29&;&R7DrdxA_3k(B7$bVv#c(DQe#sz$g)k(9o)naS&fc=nP@&9IJhn*3PjJOcc z{}FP6NZ($ca|0fy%@wIN#{a2IrNov&h<9Nrq)&GUAd8D*8{561URG%!`1C^AM@hpL z29GTJLZmpQwZ8#Y*|Znf;2$zb3#rt}m$8|xFM0JL3~D;?wDD#sE;CLsTmKU(+~!0Ks%Pn7(>y(}iE$ZQ0R zYP}ZCKYXg%-+(Sl=YF}Kq}MVEXlJ0p57_OxDXE7@Uyc)c=si64Z?IdKP50PH+25;btt@Av z|A%oaA?{~=2s&WE%0j$b0DPa~kBA}@lRza5BQO4Kmw=x!$(=9)i}v!O`g=fHG=7;EY9BKe&c~ZG-Q(=rJ zx!bGXCm&Nwlm3T50gk3SQ2QBba$Fk(Ja2G26sM9@8lnODV*71Y<=24FKBy!~ZYs~a zjz6CBBGhWW&h;l!^Wj^o-K(Zm&OGJEO%h zZ5BMY%Ir)r&$J}c1hW7e*J98AlS$(Ed(r#Fc;7s2Or8P{>AbT4`U+4w&h-)~32uJI zD+~g{pKS;x@oJFtr4sYRzehGas}oEOSODfe>-ok<(Ct4zmmf+^QL*jUQeI^I-=3{F zR09H<1cICn%OdBkfTyjiq{gczz5JeYm@R)15N(+l%WVGp-Waff->@SUUTnRZwSFVT z6va|eqYgrk$uA6R014gHtgQ5Biv|t?IULZpzxnb1Qr)7c<40;>cKq(|XW{}3(z{%sB@jAwD)4HuA4A^7&SVgMJe|2AD`1PKqV9AM-EfTZKC zTV>n3KfIj3UL7CbA4shw^B=U1!xnac$5Wg+(9sW0uCI4o((|`~)H&n_k0^V8cN+<4 z(wM zA8BgSkL+J805bJB@NrD@@!|ievQhs|>wp-^=6!Xl)Zc(38@@y*C4-xw{_x-W2AHQ* zL}#nVN%~j}3}eh?^s$n^u9TUiNG_GVJvg_KXyDK ze4gHX04R@xDV%_4`+qFizWuV07Zf-{X!PlZ*upnpwPC9j^FP*!B>SI{97`3jLFzWj zW-R|5@BPgQcubTTf4H9i-g>cLCvYHkFjQ-{(cT^)Tl!t8+O!ClqMiJ-a$qpS2EZ1V zN{s(Gxv-mpvZ4t9hTsOE=CQ%1Qed|7B3OkHEe^e?%Bmg~)1K2vMm=luJm&)MNi%X% zg8%RM0FFOElR@&ATG24uk4eNPf(@9$0>&)+Mn7W`;00{|PQCpx4(gbMPC$+`V*yp| zCM|Gr%Cg!ql91LHap1rE!B`spUw`{}R+}_$g{(#6J0$9}Tv1 zc-C#H|30GUlG)|ugU@WHzo$EoNwauc4Nc^;oxg{^+8MYli#P8P?MpWsfl3#2^p#(@Y@53@z( zf8P)AKCeam%uT`2eWa)S7CpVwi7!`JPBg z7faRpkEKc=lAw&jS5ybq?0^X7 z1X9FgzzR$O>w|a>cFNEn3Hh%NB&7kW|Lo20DT7M(CZ|X;-r7Ub!bg8nZDdI5GfU$r z{(T<65ip#B!OCYU;B0h`|DPmA8ZD+}`75dYGZb=XP$)?=T8awc5^=l^OL=ac2$=R+ zZzWv*VMT!h?tepjc(E4!=?NEW>tO)M>SiR6RG7gbEiAKY$W6D|Z@P7La-OQ(7kxD} zS-9FZlEytE0J<54WfjZ;iQ-4Fa|)7tdhL7<53yx%D{K`yjZs!k0BlVK6GD>XSAULc zfw&=<@OjSjNtK6ZFApqpIxuT+3|sPsr|?nSe7`x=zXqN2cQ~WjUq50mdO12X!VP{>PG-6 zrs zjzi$6amPe3Nx7cbZ>ypbi2AiosOvDA>Y!(Qp-hCedad=rG|@(JUX+XxbH)ESvw~M} z+($=~a;+fODe%O>*Bn53K(R{}=N_xVIEP+-qWUje1a^|m5Z;f=10vzA4wwOZexUb> z`7!Vv1Sh@}n+GX!lgR%``%y(wIrCJ<4=d^EeFb7%7P zJv&&@AyJ=4YS+|72DZzM*0=vefgh_|c)L@fWO}yfX%CaG2Cp%W#2=(aqs_rXS;9-Z zfL<77dEqHlfnNb~M#XVR>T~N2Fwh^9psxVRA<9C>wXYj-uUObY>xP}bXS2;#Tf`Vh z%*ya9v$H(?Z%~C_Tj1JKhC^)v;s+_y`L1_43eu!X|Aens?EJF>58FACXOpjG~- z6dp(8A3-l32>#Fo{QN(+=notIg~?l#%FZ>|4o%mXe-9{cehqhfC0~^J^NmrN%JMOaW3fjZR z$)J;K5Rh-YJxy(gasFC>-y58yL2#UPd!DL|!`#)spISP&k^Nsz*fB;I+bFGG~M{b%<}|+*jB;Mh>sR;`lmXv zwi0#h7&Y2HsB3>~1;Nt{Y}I1VubQ7@@P771q!_E&lZeqrV%Ux!DF;wg7?O@t=5X8-ap_7OA z=f3ym!ys{z`u=lsr>-8&aA&aQCnzo#*?6q~t$DIFQK7%S%6S1Wjd@d6_sO0Le|R(; zE5%{l;}H+;S&O_?>uivo{r4foa3b7Pe7>L_i3Is4@nqZ+DNtSPo}H2o`62k0pZyyB zTyHbOaf?uD`3Eh4?F0;Ni=+^o#rg^6ZY`0_lfgJ14WOTF zfQlHFH8?|0sKiouvg9@1YJPqW!f`SfWSFbBUs_BD!xy}B3bg^SaZNX>`vc#5Yqfg+ zrhj(5Xm{3XVUzAR$ypF7ed zK})7RwfFv8Z-5kntx&#Y39Nkru+6SxqB9x*tenq_7TqbT1@5Lz3eBtZhr{`PKP|JD z)5Rfj)@^a#Lg%N`g9Ic7V)GTJ_XQK2bWv@Ajdko`3arAZqeQ21AN}{p|KsYt!>RuN z|8e8s*vFQ6?7g$HXNZgvWhM%tLqt}NW0NgpMJFpe_x(_`F}g z-*t6e^+(2eJ|E+Lzuj-+-~v2y^ZkKsdY2R6{>x%}7tq$i?U|15cLMSli9M+>me}xK zoj@Q~jUX3Vvu#PxUMiG9JtM_H@?C4+?>@_5Acbl_=#1fJl01U10` zka)SOZP#9_BxY4BSvTW(xnT50Mx+w`?^xvwaIz)QsmlUg9vi?ng<2FZTW!91womc= zq4fh0aI$Hsj`y}t_;p#8@c`6rJ;hrwH*)}Yp=;CWt^S`XZC91H+{AISp`rA=5LI{? z9In1&WhRGOCs5Z9)Sw54g?G>NucEeMV3mFoG}EUm6Y8jqdB2Xw$?L;_j6W6lCq+<5 z({#t~_+aZnE)bhXefUS>%VD%0`%k?{e+i%C(K zI*nYamKo_D?z!OqwoE47Z2d)v-ajWZ=g*Kr5920=X%mt9ks1at{;S!}xwT-|)U<~& z-k42Hkb1o1_@&WzE6g17E}X*q2G)O5`wrN$;~&AT7=E;UA<8nh`s$^;n*t%Az-vsU zi;DcSU-={x^f$uI$7Ej==6+R`g57l$J7=NZ8frLGh-O z(FzyocMd!;r`E6n1%VFWx{?BfUDV8*BkHyRa92OD^_vSfh}DSFdNM~~YUo%1qq&7mnzzjHTq_~9_f@^JkwPQZhu~b|x0#W!f59pj?Z!l- z_x-kx5d(XA>TU*u*nC8y-lEQP1>(UsG$lt3Yu8`~NqA$n!Jj~fGj?JfP@fwI{3}p@ zM8njz_xEO2)oekwd)S!8#Alr6on0U?UMka{N;q7IT{rBB4M>2hV&tD?<@l-&JI?ye z*}iU{fy=`4H(M%HW7F|g$^lZO;EIfYtz?A;>Z89e$(V4wcB?OA7FyiL*Sfb;VVv$GBS6%DQvJ zOi>_}p)0BP2sH2YPtryi@TpzOPFrsU1`$t)x14%CTraO{K;sU*-gTQ3-@c|Z_vJaF|?lG~pbSMuav-^@2vhHxQ)GJjB9EyPY;^dQEuVCfXTNvd~*&!l{`$Q_OZ%dWUlw1?o1udI7j^n4L1ckOT#f`3T`+VnD!0+O($wih8q)U?$6y-_T z0;$C1bS9m3Hb6}D5$vX|3EN6rm6Zu#5ID{64pN)t#Iwlud2daBXUhLX%M%j7c;GdsA|C&` zakJrmbdt$G{KS|{JfR2r_@8E`KsoTxn8|n&=qFcT`b|p!0odMq4QQ?ZclViVglu;L zp2MX`?9uLuMW^`nyU_puFVVhq3Fs(a%q5Jf1|8$}t}vus78bo~MqxWza;-aBNN*t= z^$M_HuMSnsMX<8`f$Q!i#kL+1lfcZhX&`(D~Vb?5EbgiRB@50e_ve-K61W(mI% zOA21(SlldnaiiQyz=qQFsV}v9lZJuxaLy@^TZ_1*=yk{dI`&HHF1Xgpt5h*5I7S48 zB~R1s*_whw00>ZG!j+X{FOyriU45&fyWLYP4S$$yPp?X%?wH02u3|pzMk_*X1bm z>bqWKQ+Ien5>@!i!|#d5%aE9toJoA|wRRrcefV=)B1|wO%6>fAaE~TmiB^bwH4<=^ zh7_+JZ7@mL{)9QtZK3bnqdo~}4;RVi^!@nD$*(#mPj_6muimIkZ_9N3`5w?mo#=>f zAi86~UPP(@qBknU1+eP@hm^m5TVqKQm9^#8X+pnoLB z;oXUeyX+7Kx=Lw-=Xqp+-eZ=J_4#OXBfFjRbyO7SgtoBSmL#7q3%B`WU-s!BpA~qh z>W<6r1^ixHBK}HS{|@oh5sAD4j z4*CH9y6jpPTnq2RL@xJd#uqNTiV}0Go&1!k6LVl5&IbaWJdaDNxGmUDQxqISMI+*qxmp)_8^B$oqbNrMa@D=UpbB|QS4z44{JP%T-@45}HTg6kh# z0Dek}fYfvLv+0mmmi0i`vrgSkF^m=a6FZ$29d{=8EznC1@;_+cp1&%Wk1VMf$rs4+ zHsf97|1Qv7Ua$v8oeu-7-iz`HxWBCX7QAF+{~fjfR1`uG;2t!*+h*~au$>&dPu`F# z(#m)|Yu&&1%HOw3GEHk4jEnX{DuCT}^>?p9fP`>rZTy_kW7tIsX~4R()sIH9X^flR zEc5u1w9Ob)VfJ(ur*%n)z>|9B=mSd|O=!ZaM?;@313TRBtZpi9{xTAB4liiY#?072 ztyv9ZLp=cCbCA6QhbL;*p(ybkDYtP+8T(c|HNdsH#}a0;{i2zI!0hr=1{|hK&M1nS zp%L4TsP{EOjuH*RKvz*=HTVR=Ug@?EFZh&awNofbBq@+_(O|NsO{3i*zX1XI_JLW* z4e71xsF#msM(7>Fk`a{e1aL&jKKU$Els)ge6lS7udasC8` zAtpZrFvYTB%O76P$c^97V_o_3wwgm+w}j22%<873y6#QexaF~;4Q$f_GxfLQrMVTT zJ(qTBwP$OW3vJH3C_cBIJft(nSpC13mEs5y$3^4i_xIu!j$_w2eQ^%A9g+K z8D85H_-YP*gB=9IbAo{LmY|m*_mbc`e=Iokm1wgCXafZ%ukOai;$hyv*8)*_`{0s2 z(GU6u*;O}1_yNiM_nEyWbub_>^>+CzPoP2jDb%kXOr;98JY@d^XVh>@0^g5&{Agr! zHN>Pg016?)-n8*Ybu*8$9v^4k``C&mq!hvllXb8FVqC>d(AbIFc!jK(&zz zpv-!8W?3)~!>HL%kQJ-isD*mCAyRbNiym9^F}*>+ILPCiW2H+Jx)ML$TT+7bTpa zi9lRwtwjf4WZgZ~uiiR*%+P9-t-+q0DxpKRf+vQHfq9+`#vW{+PJc^TZ-++wLt1;0 z({BIKVuIh+Dc#_T`!5r`rS$z;Dnsl98PBGJ)^k~6RMbBvOAWPd*hcRC3!(hOS>fmu zT1~FJ%>fCmk6|ZN@d~O4EYffr{UOCy>(}n7z-m28@fUsyI8la?-rUThI>@&@4g9?1 zFe3^N?ock2gQoI_i>?y@3w=FR81=B2ZoPvre+-lt9CQd~97r=ley=i6>B-^o>NV*2u25`x zyisX&WIGdB!nyLeA^_N#U7aRPMr@+abYbko^Pe;@#F=37qdWEq^|*9~hVkLkmw}sH zdN$pUSxFkTHA>Al)8N`w)+dOdhXa;?>pcJ{a;wig#o8&=x77%y>O^gu#=p0qRLZX2 zdwo92z54)Lv-Br3_#AxhkHEKWJ)U9seML>5@C638X$e!jgMr(ncQ=`E#)}H&7L~bI z*polU*CzebsE#1GtJQ*>@_CX71t?i(uIz`B6Bqm&fW2Nm4&-{8X8fV+;>q3FPq983^8S0y=`8VE zIOXaAMk}{AEGu&dzJN7nx6oA1yaP)e`b;ACM!1GPF;7t9pU7J8{FK3+rT(0#s0KgJ zUZ7I92TFoDn2W9CXv>o)GDC&1Ec;GnoMPgwK@vR{@Za>S)e0kp2*O%QISsO^v9D#O@3f`H|GwDbXC+rKaV=8;K zkl67qzn?J>5pX-rFUcri1rvN_ks!1&(>Nyr88hO2ntS#_y88lV5OC}Fb<7WY@#X;Rk?~#N3EAXjg<``cVx# zVz@`#@TYbG#R6jn+Jm5weiR>F@|B%Qt3!(-@o3!-Cnj`RQJ|ZUW(1Vt^LGQ%yI#g$ zawW=#kC9Goqa+pqjG@)hUl#9DF&E{Gczc}@I4RaYUS2I4xc7A_`Gno9c9>&I?OMGq zSN8HWdG)0+c>*T!Y%X;K8u^^VMzu zs`KI>OD9BZTSOqdLlMmLh2_U1UhwANcMr`ifw&Z?2!3e9Ds$Ixkqu)0){od?v*rgp zzh=qazY&Ian`S;9@DsFGTh;*S+oOhCd`dLX4HUi00HOCv!EG64N_U#vWyapoTGX5^ z4nq-f#WDFJk+^-9@zj2a9`2Al;0b9@ zfR2j!lBBm8-MhABa!b>^oJrd6XLF_yt=eXNdsnS;U}UZJP0eA=?|4+(;Zfg!j>`Q0 z37Cavf8lBO-y~fnWMi2>P}ttcXr2uvpL*|D;1tX;<~be~>i3LGK{JDKTBe4oU2fAt z3tw*DV>9PNfWoFC%`IA-0{3r=e`h55G_yB}$1Z|KU$hY76t$|T9RRNIQh;$u@c%AR zlU^?%^e+I;%1RLjmFgJdS5`?-0h@%VSTUNFVmv^M>+U^P{@B5HDLL za%Q={18+6EI?}U_tR0Hn@IizERUD_<^iR+=c&HF3o~d)XP62~d?3;duuR01ucP3_r zpZV1L5Jfc#57{YD&oKik!7oh0H)&?2M`iUO!oWK7G*Xg0|5|C=ENA@nZFS3t$s}L5 zD2dBWU9CJC=hsMxX%q)N$#>iAxiP&b8+Cc>gndT5k{^V~>jXpZU>3-VNW%f2a};48 zLRf8|u~Y2&`?~cjFPl#HUqE1fRXvrTIK_Wx&3p9J`!2GV(Rj-!Jlm^=?0pk3oU@i&~(+Q@Zyo z$Gv5isu~br8}}iFsxuu&Gw`SH1S3gs7%V=+8)NH;1<$ZM?V+`mNDtb|iAo=?4d~!m z!&H`)!Y2e$M1J;V#9;#qWdj+^!Wyp!JCJQCLS+svp<&5kzvEJ*igZX3;iRpq)QuW*ST!WDtjY0K{a< zOJ~7EakB8muZqMerTw=AYb6@bEvF=@>&jSeuIQ{qRZY!Up*TZXO}M$lmAKg@oVW|t zoN{oLrZ0u}110i&ucXQ=owc1$(ynzG%1KD;SDkrp8vZVTS<{|x_U+iK7jPl zBy$COJoSOQVBx|8oll7du*m2CAxm(Z;Nwp%KPG*4@55YwEQ4Et4tHWAs3A$Ko$qGj zx%2_v%(7YMude~kuo6W*sZI^Bu2I0hkgTf09EK9mx@O~v1tW zcT^Dby-o|9*Wrw&j?*^lTwUlDM$B_nW@OP5kz@Axuqw!^PGQ}q>h$+9J-D8)hIrdx z?Mn@440ql$Z6-^UAxZJc=Ck`?`nc9vd&`?hYe*fLZC_a9^TsBgIci7(Y-j|^L zENVTJx?(@7C(DXvr|)+IMj_&2)$}WbCo4gjBqEcWVA(%Xr{@_aCCEO%)1^QX(%tkT;~5t_(} zIQ#TjTf%r%mC7uabFGh%JBZq;ju$tI!c!> zcs#fGgHVI%fFJ%sVcE0Sk$w2fu&TD^2qjwaBkJ;=%aI`FL?Y}Pu=?{o>kP8^F{nR- zj4$FuybHL@pIvasivF|Pc1c0P^T{S*HDkfKqozblE`-oP3l};=^ST}a3ox?s6@$9w zQ=8s5I=S10C!(d^B_&J$x!Rw@B;D;rGppB7feja3ibuaeTz04g6(dx@9V0c712?2} zG9HEEUxabbSuMgZBZd-L?f{~d;hG!0w8r8GdyaUeD0OB*0|78?N)N-xVe4i&R~=|c zedkYFrO^H+c|ix?>#QemhYi8c(Za;ah`mq1mt3N=yXAx)*vJLsHxk z*&nMJhnhS)wWx+`Xh!f!c!}##NKM{wI>PQ$CIcO!?q*aqkC>e}M@Axl1X^qODVDo+ z4rj0arRYolGyG5v>u!YZ5xs2n@{L~JAVBsgqsgDUgchMc8*EV3xoXKJL7{bFVEbuL zKZz5MVf$NBH0!+3loW*eriTuZFIwi&KZKV;ICdJ{ta7YL*lq&gVVt&U-x7o|d2$&P zm?3_n2HVYvN!lTH0vZK~OxYoXZGXEZI?=k@neXC=xh1#bW%64w4cy7PQNUtjzT?x< z3h?=Pa5++=^lFDr)Gfq*2ah)j;1BenX+-TezrON_Le-1Uc6|tjkwA@Bsl!5`(a|sNvgJ{FutE$9Ogubxu5E+uuua0KG zeO3JSKAq8<@4H`brUYnYzc~#eZ&%jtjmmAGRb{AV>2_ka9ibom;`PYI3AK7CC53m&FPt+JPRvU#rcqO)}q(BOXDOFJ>V_-hpmbVGsM zf}jWx4cFv?~CglqlvG_aqtHXvFnw zv0Yo?8eZ1Y^QNDxTcyKXlyTWD+RF44uUq9ke_DO|owNvkLw=eh2J_~Iv&mzIE0-Zt?y0;VK ziI@r`^*;3f{?kmuC9xV74aK~@!u0#7XeQBcqdGMAZYtx7G~AZHze1Sv;I!D{^*K7N<_{Oc|$$l3$C8#D=PTjfwFezeuT|+WCPTjf!40I`>0O>ketgRqbzFcxV|@nYUV7c^RiN&ryc{ z(3&6l7>ujmJ_dZd0eyj-MaC4yeS)mMk)i;nWe{eNjZ>KV11NZ&9?^5k=Q$_9Qc!TR zd-n$LaF`Jq&$#SwHeBmhO|^^Sc4Src9L<(?G8kM(AAXGOtS{vDBU@Kj!4)mx;MW$1 z@155W@JVEZ&H#IbpVnfY?j5EndlfK`8}Kn;u=<0oeai4;#wp!~@fBB>ifk@1&D;#6 zKDF@emJPzE?l6M43T3sr$DQD{7=AN5>X^~72rQ4pXNuqlV;E$Y+( zmCD_P!%IswL+q+E>*dG)6(?L*Lqnd%K$kB+$h8W3rPqJRoD+1kz{@UJ?JISM6C&Wr zfZIFAQG0!3`ntanZL3sIBLS+K^gHwXe!IpBWc~PHC1p80Rg@zBFBew#py{p^qd;pP zTFT(Aevu9rsYAQU_rSe&Sv>1qguJ#Q$1`!8HR5qrIX7nrjskTF=gL>4)vt4@FY+{c zX?O&tX4Zh_C0u>p!cjmnjONno6R6h8_!IY>JA{j!dCc0((Nj!^w_i4*_8weOFFm{p zqdp$3MN2RNArHIayNV$gB4wYUo`ye}Zi0@3=L4lzA+RtY}C^yeCQ;zw|@oj!#+!7N{ni&*5 zpB%h;f6fX9&v#43W+Ht}$U1#*=gql>r=(GDxh)m#|66*7tB0%z;4J5F0bew(y^J-G z^zA)#>W;Rz{&xuX_kG_B|D}8gz}<%umz%*DtoO#3K$_xIP@tx950fEg#k{5q?*1Yb zan;$TRgde?k`1j}`r;WUEFvOuYc7c}x&4X5FEu@fZ8gSYeu8hb$hSWL>eHebgLUf{ z`wh*p-5MTkxL$qZ5FO~H*M4&a=;iz1YJ+mdsz;9ALAr6FJXCW!$V~dUNlFYI2vNqm zQxewY**!Iv}T|oOGdb=0A|SG(;lUhs!tG+8-!^VPi^WOM<7W^I3(h9*iuE= zGiNfqJs?gyo$p+2m`LJG1ijzSBA_=JWJunJcNlzeJ8sZ+5l4KTy`>E~@PmPv9Dn-$ z?8HQW4BECT{#gkzRZ;g%Uoq=E)KzZ8n{DK65Q%K`mRVq}ePftJvk1VwjSCUYt8@*h^ri)B)XFe`qANk2fTW)OY1~HGz??xyQ`Zukn|DZfV1qJ<+3E@wG{Zw{G2LVRXmB%>@pE*!58Wb80 z)q%9UT*7nIm$Kncz~n47dJDqoUJ25ZAFO`qh7=?f6vN_)U2%JEtHkpNZEjdQ&(wo< zbyVnUDk@F*U*RZxSDxW6AtN+J>4{6)x^7

N7;1E2gasg-v1l)@QEBBY0zZB?4HIrHbc9eS$mwV|IT zM=(puClJXA1l2J`RanCvMYRSf(C1Qq0pd_VR&mme#1TliBYO)@(C3j6GD zf8@W7Jrg_uL6EqS}Oseo{Rtql-l{ z{wCnlzJmS4J+K5k9vl&ChqaEu0je~QhU6q@xxO<}83Jhu89)ki_N;*uasCl7&iM$a zw4i5@+QMJ5=q+G`Jl44Av_5!2gf1LZ4U&Hvi#LRpO z7a9>H`OG9L?D}cU?uk2!^#rIY1|Tc98Nym2imBwo94@^Wl1WbQA&tTL3j(}mW7R5y z*Wi=TDD@%8*z1)4Qv1)YSyaJi!`B}LSsl~9X~Ku{_+h*7zVhA+|s|3}f~GianYV@H=3so83A` zNPDW`0BKc#o+y0%?rzYx;a7K^M6}6l9+?SIC-p-koJ0|8PJB5BZf#q56~f<~F)C2c z#tQJsAR~d=HBA+S%$(c>U7yhf;7jA>@?-n}5YL_UcvdjAk7`Lglc+oF;3>?8-yKto zvQcnTx~tbx1dJaE{W@*1yqMxZX;A^z;qcWnR<RH7Oti(&^Zfmj(a!}? zkn=o#6kGD}I(Y{b{2}5y;~PTqb}DTF07}_ws^bVAle{`4i@FBf5jpe9R|)tffdX}9 zNSh8(x&dC>(k)D}TWbjZpn1s@ff6X=+oiMfQkxa88h!cul6z%#=@##{WRUDvS?Gw55#H;-|EMVe&ER1DYu-yQ& z=Djz`!C#@O_p|2Dh?8dpesEw)JNSNoP!U6aKbklT6{1-z4{}xvSmP1VOtgBvx_!!j ze`oN`Th9ehfJk$x4p^6ZYtxgV!AMLgk~f!}-vDDRxM8C5Uy)pCZYxOSzJm%~;j_#h zVc<>_0z@lmJ`0D*9N+*7DLg$AuY_Xff1E<5=d*r3D>=RuLoXmD;Hw{&x6yb1J{(8j z_+~|4OU=Q5y}0@W1SE+&jn1}VVkfwSI(9&nngQ1JCuJaJ6xU%Cxcm+^FSy!s^_ujf`>j8RhA-^2f6e*(u+-26`e zz+mG~hhxaD!g-YE(}~KDjGhy5Goo{T$k98irbb#o@43{S2pr_z2qeu6`q?X~{Jgde8J!Xi$Rmynh;! z^%NihkP2mWrlkc04uSdhh`eZl?9ofp_LtmiLk7=x0DzVWxz?A5U!y(PTn&p)_jY#q zTVaGY6F>o~v64n5Te1PDG_!yE{upW_P5w(N1++O#S3AwGLC-?(Kh|JwnpoPEDE&euv8_ESpdgNA-wM3*3$&WxN z+@R04aAgFk=$i*0X1vzR2VEgixE#os(4idDYy7)@fRjRQz{`L{AVbR=eqH!H9_a#h z*4?2O8hj#prP;|}1+Nn{eVYc9buXa<5EgVlUSS?Sh6QTg`h^yg$s+%ME?4ztIvlXZ zhO(Z3nfw;}t79El{G~O)^ojJA6xXo|_JOf~qJK(xssM-Td1cF=iH#Q^GvV%HPm+0^ zOqw+U=50u?EnkxStQ8E=TyETQ;k)MmN&2K{&~eatFV!$S@e5RxUR3&kRRi{RZG&z3 z-=|a$Apv&W8Zdca`Yyja1Ge7ZgEA@*lg~=E%%1nu`r{)G0sYSiK-rXp`aWwTQo!^v zX7iH(V4Z{8`Qlj2_6@h`QlE7K=cLkq`Rd{NA?d$Wfw$6|SfjRKKz40+3v;zGgW)21 zownM;$SmZYf5Es_&>-GnMKU^J{O%%Unx4=L%Y4Oz9n zcE{~fn1??SX7~UN3-#^e=h zwbR+}ToP5S1RkGtm^O;N?qs@iz))8bUKc@LRK^H-<33S77kHR(R~WP+l|bG6C3JF| zO8v!Xzjot@)^D3nziXrf&=xsRdk02iXWztYOd`|?2z2!f@H(k{bY=mC#>ZG?KNa+0 z8CK5u;SI!9JP{#W_8?an{AD5!`!#k7nD@As391bq2hQD5cEC%1tAf&OSeMm(B>rZt zvJa8~qDQ}UuYj^^8X>cs0g!|=tuqJq!JIMJ5J4VoL`)<0?_^;V_#jzZ5=Mc}rU^+5 z)1-Yp^)xY62FGY)gD}w?246bIpX)c@FvYW_w@R`QaR>|nJyrboP#6+wkDilvU~my& z!3TsTuLD3)JFsLkO+7I$5{vgx251D*PQV*}-%e$qbax;@z1YFf)iWo0KI-NQ_fVfx zLCzZ2NyWVYlC;FI{&zq{=hWDvd4YfwWFiaU(<_ZI;9m*|bc0+Uv&MgrC9q5V`>U?S z;H9vay^zv0bCT53yEGd@I0YSBVc5PGZ-$KGT7&L%;m}H3)~W4)6YyE={oXJYEo~Q! zPW*dDt(45a--QBwjKtF?2ma57<88+l!dKm7jw+LA7iAU@>(YBnkcul34wF7oebUjG(z0tCowh1p&Pm5GjF zShtLu#g4GJ#MHQ0_NQDS>m+b}pi?T*dj9nz^rHTLUk%h~5b53;1PXRty;6i1`~qZ? zLX#}qF%T-RMgVl?87J#@=g*%Hsh!SIQ}OZ1{Qa7tuP){!D9&g;m<|A6$=Sj9a1g$? zF*)`W3}uZnB)L{HcArv>bwI&k7(e>DnbT!?sY&$dM?p76QWMjf8pcT61Rlh!gA!(?OSrZwVwftp`uA zP#o_KyuxNJ-?e!i3K(IPv=6Zmn*kEFwNTGDI>3j`L>|OSp7>v3(vY)a5mPBghtMS7 zT2Pc#PD3O+FH&);P7SD?pHdnp@cp%Zp%381Nss(_)J}e)`2ZnZ180Kzh{pMDI5P?8m3fk(ne zK*>ICOOb!^^Q~iG7InrfV+mPcHd7~(R1C0@lqLer!uv)L`NbA? zQAX>V&A-q`X&cVD`u+Ks(xBMqWt&pjNnFrY^NiQLLl#bm%r-#HVdrH z-Q2RaQX~)W+leo4_HKl>%6w8@FfscpAQObtHhd4);d6Og_3zQH-iSnXUAX7RaBZsc z*O{yg>^VF7soX?QiYw?lpB5Sj_h~i$thvIp^a*6A3`sqv?!DK^VBy_zw(+2m?>i)? zT(K!!kUPseKP~E}@a+YuZZ?G{P{5c6NWSZ%|4MLyKp*}B0bsqndMNI0Q^m9+NT&w{ z(P8&aYQ}+jZ5}j)4Wb%ihB-8rpa?m8I5Md&thG8`_K>Fda^@lE#Z_)I@wvYR-kpQM z-1!ZamznI>tNSLlGy5i7F;f0hb#EJU{N|&Z&xr8*;rbhYYS**Wz&>J{c^93-Ac#nY zj5p zcBY0wE;Pn^w{C|s#3hb!B-Cy^M3;U7J;U$etST{d{0gnIA7=L zjpBsA&x?Ef7|P!S+rDDYOnxCw#-BU6LB2$Rpz*~9jrwPJ`Xs1<8Uz}RqDvjJZt(XD#J=It>vm^GZr9#}=g6Nx%^9=bWJ<>8@ zowvgEN4dvtkA04UPW7{HH=u4`t(yAJD(m9LbL|97eV zy&iWo!S{O1<+AO~L;$eBX1gDnGc^}}0!ts?rpsJMk)+Llr&n4LlPAcj5#kURP!P=@ z^8yN>oHDy2W}ce+!_n>DbC*KT{^*@vuY$yg2_h`5_47q{@aw-9kWqi+>T?31ktv7_ z#{q&_D%HbXy8(4(Gx(r!rXI-6kI(#eHs9;@1XBcf}`J<&p-&F z!2_V?(n_VD;@EDgvAkuIwlr~<;+jja$w55`uAH*kv2H<_G3_ILi2H`_msRW9w9s({ zrxhwS$+@&VlTM zl)GVAGSr(F|8DEQ3%4a2g^k)TQ(%4nG~_B#zQI_ah;ZiD9ENI#j72kywbbH|m)}cY zmoEeIhmP9krYW!fC0#Q#B4kI6S8&SyHZH^pnDu@KC+-G=D=LYoO{Sf~}+`sxTeZlhU>*%J?d z9L$e_l9F{sv#=L~)8c@%G6x4O=PJSE=d84p6=fM1^-&%P%+IceoQhu90I)@0Is>)c z4dc_BfGJ;aXFKQC?jP_Q)q{TXF)gEZrr9Il<#Xss-p(er)doCWOoef32S7&YCyTAY z$KWlJ5;63V(ak8QXWo6@OP3}0-TO<-0(pVQxVQ&{(0Orr$uP@)FPQEBFcHZDPxacD z8xNGDp4gg}nY|oD51Fjx^qE<>1eRBo%c+6trLwXg#rn)KRG@^74EA*vX=&c-L{Rrhf+5L#@_w?{xuISt(Pru6+I1QeRB&+(2cYUG0Yj1 zCtp(T`Ha{CftbEyAUp2X1Jf`2iC>Te!|^z+_7u9FIF1v1U%Za6u+2*5%_KUd}KiKSGLs?%jl|h%w_Wmse>` zuIjR+Pk!V_P61SU87M4sI{LXjK<0E*n#9^ADOWG`&cybr>rp~Ty^`*fsD@uF5$D9e9VTI9^D7%e<>mPkI8ZR+JxuQua8#JNe*QO%wsbQ9%}d9d+8@!zAV3*X z{ubTo;qne-)A}p>F}cbPQ-pF^?^lca#YbS#>iy^m`pl>hZBDegN zX-1b<0C2zSQ4J<+jee;*EhW^RM~WbhO*@&%hqlzY>5jh7JyXZF1lKR+ri(ZEFAm(u2MV8_C-Gyer;=p1Mg&X;P3VOzf>sM$*7RDCOtokX*BEgCsRPv9l#2`Qo zNTTxdR*k^|`|sZX3f@`9B6k?1-JXb6AdG)MnZMNOL`NoaH$wdlPiyHP$hueJk<0a4naBuxbSWvZ6rnUi zn>;nvpk6xEIwU}Xv45v-NRzxC*m+I7CdDpAMlV6;B57IUxehTx%9+Hp^xDr9K7%`W z5CvSYD_V>c%`|S^5>`>Q?`G$r{gn4-PvCM+iequBdamiD^UJ4Gj3CT9J~8@gkJwF$ zCB~^RbP>xMPaX z5P;}J-aQ41)lD<5+I_~+5>QeO8B0fMaA}(ia>;8<6nuPjVT<(1ZRYvz(Uhy!G8UKy zQ$d+v;wU!| zGP~JwepYE=gZ{`^`bbB<^me4QZss-R;er@C_--pEAV^5OgGzA`Vf-g{CIA>l?QP*1 zIOc~^cZAx$_wU4<%#4nhN7I=5AzpO@L#rOK>JHk^Y4)+)tgb$S%j$=wO2E)ag)4eS zq_U~Tg7eG6xp&V_p~L{&>dOl+KZ=Psj`s5BO{JXAV=uKwVp*b+M(h%0A14TPkHmth zhdyu7YS8%C#s}v#2c^nWOHi5*C8bSs(cvl z6kW!J|Ifk8Oz@Zs1>W=t!(w4#DpLWY@p~G`##XDIX#czJx3hxeH!l(HBS?=b1S95R z7;CN5m5Y$=x7NPPVrMY zTfC*Zpb&p59YOX{Sz0L5afZAa{ua)+t*+^y$u?=4+jizveoI-x8nZ@Vz>Zk10zMf% zQkIC(K9+Nv!NlM(?-pQY9BN#@c{AdzUmZ1+Cxj}aj4b46+ zmS2?ZZOzOKfH7rm2qE~)j4*21K`jT(n?$D>I17;#FOv^W-^~)#FNR6xcV&-y6xjZ* z7G$a^<{;9dpf$IQ-)$xIVjeFMPA=q{+Nsb`eQ8+yO6866y6}68uVPDCoq>pJzmnTe z4kOi0+ZwTaRMR=#I@QjMrsU-78c*|%G$%CjN0T|D9su;^CO!N9r7g4Kjxg>ETkIV| z%}h94K}8rsJ2Q+qbB|p}=f!-)TddNSG!4GV9MwXe;6;s|u(DEO7xO1t!|TAckox6* zbi|$h#<2U{7w=zJNqv^8zE`B+m5W+6quQYP=fQXPZ0NoS`0Q9 z6`bgpUX?cVNNKB^cwsBT^t$J7nfokeu_vV8Z|FzOkQ~Jv2=lV~j-73&ZeRdhF|l&< ztE_-G#S!wubMwLgn+MKGtpB$h)x+?@cyCZfhy)j;yi19}b7#6V<^2wTZ)7Ps{|EE@ zbp#g^;74s4g8fjKt&}HRP)NHcAC^X=&C-RoNwfd(cV`6hikow5EI9$$ zmH*9ju`uE&N>?b9`p)q4OUVJkig-SvL4R0o)ZutBqOf{0CMW%)(W+PPQaP=wOQb&c z&sRupC1O$y32xl3w@=Q{50XuN;^#X|!A%v$PQf58){;1*S-`vkAtjG!%qiv*UZXsl zvrU^#X+LDJeciELy98I{@{Fu1KHe6OAQMLTF+U8&7JVYmMwaBq-}#| zf2y5Fdr-FF&oh!a0PxPwJDXik48fM1su@FY_fW$Y*=q7`1Sv2VK^86}%BvcI#UETk zlfW**rCg_&XeSm6W-+CRuuNb|@jYd}cwq#^N%C2wwrRaoa#QrgATUfFE+n?u9;;x2 zMuVv-&AgGb=l;7}yM+1h8dlx<{AmqSv0jsThpB`BvpRKx1H4`{SE1($6JH=-pbYVdM>!eXKF4&h>o zUCjEBm&s)!20-Rg#(Jv8cwzxg-_uAIu96QN%}!%~!-_BF;6oSgsYi~wfYKE-@dBsz zWQ4(UNcZ2&qS*h{u79D>fIvt`feQv0thDN*+9QO0LGtL?9QKZH5pA8~CHs3IGT)`=!IaQWl(?=oUEMjcptz_!$?OI z>Pfl3<`FdlJa4)&yWnT>oOnE`C52&IihHuB2T**(!PP{gG&5m@jiCG&_asP{j$)y_ zt+kYn4G8N-;qT?y>8XGG62|*l(i%r?%5%NFortNTYgqK7S{P#F5_5u?CheFeTWm@^ zl2^)ea9{sXrXlWtru{}aqc46jw#2WAt*8P6M}3s))-Jl=Nw{;NLxzuBb5SEGqCx0l z-}s<3BLmACDm8j&r{Z-x?#)jTG?Dznk?2I}w8X+OY9`wkeB*ww zrbU_{U&(PsS6*$nVVDC$#l%Qq=3N#$YE8U_o2E>rB+_y{yBOd;91%j&Tl>sg%T=dW zs8Txraq-<)+JXjcBI-4TxCSBbjS|mmb>MM5&TS>%HZy;WEPPeZ<8fUjOw%g&#(*!o zCU1va+#{lyOKYBdTV8B$+f3Z4q!UVa^0BoM2#*i%x^KPruOG$Qv^9pcb8gaV>0p}} zmi6xN6^3B}lW%mFP`LhUI|~69Sde6&tIn>&oKeyxmtUojq^RucK5G9D&W%(Th#qxx z5g~1~wkJ_0pvcYvVbH-u+?cEXx&QxatkiAcvscm4w|RJ%!izhvJ%IW5VwrfOA&hsr z&fw{j*X8ea?tFv^uBb*z8t%A**q+i3fQe}m(Jg0x=+rM}!@C(mYRUiW`Wf1<@4}W8 zHYb|@xuBHuj|n34r;x|M6$Sdc85z1sMK`qGlh*0GCWg;rnY-57u2FL13mYc}n~<5_ zVavCEatL&QK9I3U3v8dn`bvH+76zk_Y*&!=c!$%Sd^G9)Qp?amCi7{&W~f6oO9!)g z_2)Z}J+yo;b+B5f^?6CqC#AL)kTJWAh5tXY-a0O-cU>P=a_H_H=@gKXk}gpgI;B&k z8${`bkq+r@BqT*T1*8O}OS(lsK;LJ5_ivxQ&w2mnGt8{@#C>1)6`SdNNcS~2HZ^?$ zIx)eMk8&(82;lLzDy$9Zy}Y)070knXAbLaF9cxzzS;#nG=M$J#;GkjrdVtQWFvaa& zPbke%ad{HrSGuvDtmXt>Fax1s60*x2M%`O<#CGJMoP@($Wp@Gm#3EpUghT8V2DMX} z(@~fk;w+l7%5NC^+;H)Na0^qLWyw9{wp9r!EPZBm%*E>Vx!`5mbI0Fg1vg&lC5tZE zvb1l$&A}6%>->0gIwJM3c~+jI_#J|QfL=lr8-pX^XW9JVcE~=kL8^;DdI6C0^7s3c zE$8Jle#hKWck{*+ECv<4zujV37lymTNlt7&1bDLKyDq!=37$Mh;$cAq){;ZiIs~h_ zJn9v{+Ma$aP&GHp8&+Pynn)7HUd{$RF1sqP2j6lch=V1~?R}b1XBfHCk|PGZGZsNp zx4+mZvmpH6RpiNu3_LPkzddlqTtojqD6XT@9a&=3iqxTUW~e0^p&4@CXdA)nw27bj zKMg!3$`}=VNL8i+{)VRVeu(@7^Fk?+yIjd7p^PV&AX?$(5%~*B7TB8y6zdX)wca0Hc+?L+e2t7c) zk-Jf1cqJ&Kg{Sva?WRH%A6pBUH#i!|YEr^^BQ%ICxgQVN8ywR0zx}|FqBHH-t@5|w zdG~tbT41yQ3$yI!E;6eH4H;FMPwSEe_+8z(jk7iBnYmh|@5BHmcSSr8hV~Mjx;%T<6BqavERMfwfi1u9sWGL;jURQnZyt9Tv0Z{tIsNcjz5vHZ`+@SS z>Tqn86K}cM7wn8Olrz4h7JTQN@|HTULM|hX3|LNvg%Uiv%zdGc^FLS*CCb%3My{XO0;Q==g>;0OrK)6rnx{ut zHjh78Wj*nqqYl)1x#C1ftbri41_~>bYH_FM7Xr}d8^@CM;_1!IZn>-i_fDKOVYc6i zSS>+n_-u+iyx*|%xk*Ad|7kSYD@(=Yw}hNzha=*!%zMKrl2d}zrpc{jr*cS$HD!!n zl7%!NR-`u7v^DZYt!3Xoc1}mejp3}8k6bz6N7N~hve}c>#l|ZtOQNUG#f$|;Bc<|< zVDPf-ccnZETM3@q_oHbVx2gI1dOeKXA*Kn}mrwXlNK3J0$0EA7-N=Y#NYx^qDQqdY zaER%})18*@n6&}k2AOEe|VAVE-V}4^*SU?dJ{s>M2r6ELe#G5!~+xGAr>47TZ2OS zzPw9pnQF$3V%~w!OZ_if-142fM8E-nKU3iBoEIos{c{bvJb zg9PWz#i*6_X-MsY=gqoFlhM2+nh}4;B78LvqVW6VqI%C;LzBq+jGm{nt9p|?>iwvj zL=LA_;D<$Wi{UG_W1h4kB8m`{9S`llmin^uNY2pL{AlB~T7fa~l)T^wb|JB_Bh}T$ zz{!6`ar+~4#JH_=m`uK`47j6}{;{FYKC&zFFuM8Ub4#wNEi-5o;4fJt%9A}N-$|NK zUhL}eHMI7vi(j`xLi;NZPx6EmqkoTkb6wH&s~+kyzo|jaxP!t@8Xl>mT8SnQTDaeA zp#1N@`WFS@6qftujX0AV2qEOcnV^N`dIvvd@mQXY&|aOyRKVBqf#b0U=NbmOb~a2s z-D6ctThYiMI@Nre@L3jzXJdxjIkJE?aSdOALXA~pvDUn&-6CEMd2IR}7R3jC8cF7g+v~Bq&uk(AC<&3q>*%=2n95*teq}VyR8^1$lJZA+b&ZbGJor*dX zKWQBcrX}AJR<;YE8sX|-O}lHTW8;ZDpd~P=h5Xn%2=Ih^RZI=>ydE?kBt&W-@v5^f z(WyRh0St>BTE}4x>wCC|OVb+sGH=VX!yNFVbR5{F-eOE<&1k%u6zCNKGHqVt-+9C? zKr_Lt7)m`2Y;;KenIbM~YdzTv`MJuPD&DN#YfZ>0n^#fa$h^Pl%02XoF^aWR z%A6g&tQ_Z4&z2B~DD>{*j$2Q_Zc?KhKmYt-d|y+FgKGmzh)HK}g-!iFnfRt_oexBI3KPE;Rdc5Py}W9cB_5BI$Ul^QRn;$^d-a2I zGtld<)@;P8pVrGonpRYtCG`6^BWruytBd8!HQRTsBB0smf@xru<5(wj@-S_jrHlI&eTCt9Qh z1gVeypBp(zNm}>tdPt1*z-s2z6%Ka@{EHcyb7+s1og@C7m8c<*YGzy46m)5WFua&?K#9v9UswE0!f zAi<$qT6_PKywY}2gz4!PWArgDpE2fJ+sQm(U*?pjC)Iu+rmjxu*9a1ppUL+BwvT2-uabW$iFOteQJlk`lAu0FTo zDYTcpMvWtG3hzre}vd41Hp?0Z#jDFf{Poqxn?vSmS z{?i{@4LvX8%ga<-(t^2AUTrRRVc=4(*!8edp*@2YQHH22#Pn4m&NU?H8vj)PXdAm= zanI|&9)I`nvLBBkpU|Le_6h?r9v-4In3)eF|5UjX0Ls$2l0+~1g!2FHiScEL@mx5K znV3E1dYbnC-@HbU_AUn6_R5@U$S{2$WOAZUCvPim&sS47$KCLMdg5a)6fmodVZ)|` z@93$$sA(b6&_vk?Rm^Pudg^fZ=Qq`FfGD>+qX7klsGokRRjZ7`$xz0^L?OJgaCHWQ z#%NaabhLJDRvo|S1Wlg)kbA>K@Q1JjtSeX~J)Ed8<$9;Fl=7hcY%E(ae_(AM6<@Ry zG6hlyhttWV1u-MWI&odmqC^O_Wtp64)s1M;?_%mu!m!|nNkaYi)4oWphLGg5M@HXF z&nx2ips01*l6=|%DvTeITxu{mGXtO#l_XN?f#&)h`!Z%K0;C@`6p2!8p1CNikRtD0 z@;WbKb5&}%bIe9$ir0{bP*Y;?6Ih^a&^I=?lCpUt4iVqYEllvN%c0gsnFz;~nz@lX zECDl$5~(@eA5k|0n4xR0^D&rt_av~wPa~-*dwB+*uhxHkj(Bcvj6ove_Dn}(T&HbQ zQr;eIGgFW$YeT`9;rFBL$b?{rE>sVZHBDEg$y<2-i3CDU6O~I@B}rv^*SnVlPUC1@=>EWk3&6*+Md!J9 z>f9UXL9grZ(e^8nR8Cpe`i@6fEqn&qJNM+@Gnqhqxi?+Jx-@a1OX_ibX5?uw6O8sC zq_9v&va!!m??1pHG)Ht7KmmCC2{xso^6rM_7ek+L=593OGL@UdF`~p0wa*32cZTjH zVC?dgM)Tc`8=;3$xIlE9MKHFfr&|Iu!+W!4PVFaQM}Ik}&=c?+lNs+O)y0QUp8gU) zy1bxH<#tImV87)f%Cvzd1v!SyXC)U!*1N5Gm#=3Pg4nAS$X%;;zeE7BZbxF;dG&;;rH zaB-e%K)P9ah~tozJ)ovNK(4?0z{FURlP7>Qj8QKX;}R*HGimU11)cnO5GB69f1erfwboV?M&b{fc0^beftKm*^So$VY%FAkB zLVRQOUj<_!5n)^^gP9J5kG0Qv`s(k){48(6w~w7;PNlw>fpV)tTY~3C_`!C>HRXIO6u|K=&ISRMy|J=vH zL}p)L_S#}%(0p5mF_=ra3XHME9myyEcPAr(!v1O{yAP(a`M!>DH1Li!nI??nq{>k( zCI3fWn0V6{I&zZzpXhRy3FKaa&G#%$34q#w?su$OklY6Zjg)^g+vK*>2STp2ceh|d z(%q9Ss*sb|+|{(yAG8X1bu-JXLncO(x9ReILKK|$2#YC~_10uGFD2p;OKHxbUx>@) zHhkhWU1ZV%lLSE`(J z77Gg+p*TPwyDyR4JkI>k8fB972qh$z`NC$kk8tAhM{VqaS{{+Y|(YG_dppRnN|rZ`K8aF zh%DnSeUuoMPb%f~@?IXizcKCrt80G*YU-8}a04bI3!edSF3UIZ!W+x}afl=Clp_ie zi7SJnppTw!E_o;y1mzMOr&TS74 zJZpYH6r#~{i40*OaBO{7vc&p-B%|#+fGGRkSLxMjFXCEqgS1k}*CEWU6u4^?Yw@^#lPCeFx`uC%bb~`%FvuGU4=8Sx4DY=A z7C!3SlvIctC_uuA`2vC8g-*ho*E;1~5-NZjmQOG&1+_1YSm%U;c96@r1Olfh2_!1d zhVsb`r>PaI$ppyZal5|{D<|6ETj+mTH^k})2q3(iBPtK&u6Dt={>%z>m}52<&GJV( zF_u*d#0*Ik&>#=A3w9tXY__TQozaOMwypnxd0qnW=ANFqIQ9q`V%#c&DZ1kK-+s*( z(eEA}{f20KIzJd6^y8rzkyNBA=ynlN&uu_q&VoA_k7||EqJj)5}L#NNer}Z z1rs!pagNg$j~;LsjTJ@Fcp=A&6z$2z5hG1{5RB83&1t; zL4xK#4jz`W1Wg8%@(IAwkHsdk<0lx=GkFB|N_WX5VMC!UO@I-Fp_FM$(PSr~H0`Ou z92gNw!(abZ_RSV%p2v6l;-SLY3B#89Y29Gywi%t#6b9o@Cp?|bRUd#sA&XS8&0{Z% zdk1DCOvic65!PdL)=k_PWyB(la(#F^+#I~5^}Sk8pa75MHqMU#8f_)KDvunm-Z6&Q zF)iaA`S1K>6H0Lx*oa+}H8TG5d!#Sm(&3j7!{?U{;bLmf?*%N`NoP3oOQ(>+-*;ZT z|8JJemoC5rFAHV}h_<0iwP{w9Rrx@NE5Z(?*6DqOnH=?={Q_jwR>-=F3m*!zeUnu5T82x%uRrM(=R<*9#CC;PLu)&oqDlE#wKU0JmDkhbh2MUd}8&)PmN^aQBJTYeZH0$J1{JbjwR@9*v657X8k@&jrO~3P`ntG-aCA6Zy{BM#OB}!K4ox@z`D=Xf4vAvci zSI1NX8!C}l_PSc<0<8l8ZhJz;(LIs`R?MUYhC>>TnxPedjz0oPzN?b_A6I~C!Ad@N zX0O~^m?kR+i~_$){<&P2GH5;&Z8O{v?TT9{P!U}WHsp_WJe ztwQgM9eh&Kn z08l4M>!E2_?@yJ)=Oj`JztEq(1)k_;B_khOdOL!@*EAl=Vv+w2%^M48e($a1c zO7vGkyGLkANOz!xZw2l1r&|HLhr7biT6(F`^eYx=ZF!E%sZxxb{ysjb`R}jp3zIHy z0UkRG%YS|~eSPeYs^t$zzb#PK#rd+MIN&lKKb))jcYXN}byf)tSmHYa>e}){EoFy+ zlKE^OLl8Z3D}Rv2Z~6T}@XL1F!P)#tMRvzn&q0J~Ce5SyO*FF5v_)iJ);5b1f% zzig`_N~r6xf&cf<;zJW)bOs#n7D#3ZQ=zJ@V;l|a1eA=!A8D5OT1zkE1`r5N`d2>@ zO{2o|m(Po{eH}zVQ6MtJoTyD2jxG!_o^nlyC?$Ox<^fMXx(-Btor#8!Luq>#pl1H< zR>}@FUJ9%G;ovJ4GIjcejtDEi2G8KBue7)&EJ+4z4eL8~Q-9^mAkJ}Rwhxd%#3HRi z|7Vu9ZG)zv)C&CT{MSEaIxS8%(J)16zW(^j5D>(FYBlu!cY*)q9gNm@JMR$K;2(3c z0It2Y8|$Q|H4F`Q^XKA|7of1Ewt+(@xZuM)HjJ+L^y=J$UkM*NaiI8?64< z@92f$f@YilZrD7~*$ABk*A*%Neq;t#MX`Buk33ssBl+`aYna@5V`- z4t`Z`F8ZgQ40M!T0U|&bD4(beTz{{wxdY}Q{(I8adhVQ)TrMsOqzzaE&D{ZzDPeg5 z=5#97%M!dmcknZk;$pL~fOS`HaQnL&q68F;7yob64WuHhVBed7rGMCk$*^B6{$b$d zVapc1i0g(j5NXU`=@cHk-4% zU;E@uCy*~vw9txyNSQ?t9GbzXla&|(?R%iE{7mP)66hF=Cu|oTBe#@G`vja3&(8Bf z0M*O4_ebRQ(7wOF@f_W(QGzUYnt))v8AN2sV*+W2g}V0cOog8NXHRI6N}zy_zm1I< zLemcc3d{96Fcdh*K${Ai&$F2_5IFM8ja-4rJ|)Gq^}k=3I{+CHASu<6SflviEbj&Q z^EL^U5oj_0PdPFg-U+E?z<8AnoDCgHmK>@wW`Kg=iNn)?q>=P>SA~f$RKlSAo26Ls z#9aVHMYquI{=gIhQsA0Ols`4?r%{=J+}HXL{c8CT^g+%*KK4G|8;oG^6RHgS?tO{d zY+6VJ9v3wdk;lMj-wg=KjyL^)Uk9&OdX@+{EGba{{W3Fn@B%#9HlS`{@%$P@}!BJ}?rBt*LeDbeBrZ0A5n#TD>xe$m-S zbV0`l*er{H2t%YQ!n5T&7mVgpQ;J6^pK04Y63 zPeGuHBHPfRDI4^9Z9;d!(e9P_L^wUup}ef_nZ+u2qm0sbWG6uq;3`OkvG$r&VMDhC ztF#pNE)%m(nBG8G7(5H(0-q#ks$;-7=gtQDN7 zpG%y9b*l%UaXM-h{{qselc-e?7Fd=Eas_EQMF1n+L94jYfMf=|e}nfQ+^J9w!a9LN zz#5hcq_+?U^~gMrUrLUnJUB%cVV+#eyR|*|Zb0kg3TeY2F1XP{t`twXTx+5^Kn10x z`w76(t_gcV(3Av@$$%$l`sbA50@MRuPmA3Kcfi@QNtOT(0Ms<{|FegQP@p*y7b$iz zQadE-*3JeE6p446rZpe8fPDr0eme;(A+R8I+y#6pClLUd>Aj3#VIHoGm&MY)(Rev)J#%mW`D@=U4 zyNwiTLu6~{`GJ%OS~_|E(Yl}pI;+@BI={!J*YG>=9dN0+L1Li!`)(eei4JX?npZJ^ zR*tEkyz#%!hO8)1_=uekH92;Gk~5$QTw|@!=Jt4C&f)YJEyXmz=VFL0=d%5P-9a9( z4~(@oNrI+9hT;CyG;RnpYuK!jlO$Ajv;f7!)Md@Rb;dDs(9F>FUXp ztaP|HNXBykZg=deOy>jO`|mqHvW03saAJUnz6GeJgVc`u6b8yijwxN9y1@|O5*G2_ zj|SADNE#GKeW|DXyNBV=3P_t6Sy*d~zzy^T>b^2EUHz_#!LNJt&4iv}&+X$%t z3PvE)! z0m}M4iY^^$CCMI0-{_cbo#Fya4lLbi`n!9UAB+kn0tMF|?IHW!wjO+O zApRA9@qe%+MpW>whwmhy+jFA*NVlbL0`IpC7(5BJO84V|4Dki+LH6CadWtLGW0?9*q)J!NdZxw1HaY$Cw}L z!=y#-b-wEX^%Cu(OVzWF3lUt^5dk7S`0{BGlEM~xAULHq^~K5r3=$60EaOh>=iH~B zEP}*BBNyMpwj-%h9yDl*5UZ;KlRt(=k84{V-*$x(fX+5aVP4t zhd}XQdSST_h5yO1IJ{LLKZzc7d6i~0svD3vslQ8U{qBl$AF_M{EsA5=isu+OCtT&t zlupEI>)4qQ%k-dne%Sv4;Xw_S&Ta$SGwT3|{(1lPH(uWPL%>pL0u_OhU9Rxd5Ly}l zQGA+&v0+@yKfn!kn{IFPoR+DONoXTAZW_8*+;)=R{D!V*liX3!^_W8{_=He>Rm+bI zNGpD-GQAZ!4LtFo;+*uDt&=Q4lxTPK=rAsZ_jc8-fEj1{=;4ke!y!1cfF3>tO3Q^(mp8P&pQ(4 zyR412pE>}6YP`R+*H^GDnm$@>4uRW{ip~tZ8x)K*w_|l$oN-VtM>3ZP)PV$FhP?!- zEr-Cwan#iavI}kl4XmAeQ(F}?Lx6wd7*K&e%NDG;vGI{gUWnd$f8338jXw?aFguLR zc?lsC@5MHgv=%^ODX9XMiVCJ{LGa5ut(Ps1=lfoOYsR-&uA#EE=T?z1|^`ftPR>sW+l&SdB6h&Or#AIos= zr(1Tul)zDW-spNl4@&Pt@EFdT9)h641yDh0p72oJ2iMfA8xJ*sBe2!T1`y-GB((Ug z1{Av&E?x(g{Ty3p9}MYb=(757S1Bg`HF&^|Vkg@tKq)*BU!WIsa=hAO0^F9`j1zPN z+R$E1qCypmA3$dOcNv#uy18sOm-1X^=Y7CWzGOgYRx<{|+Z%}&^+b9HRE^}&MO?Kt zQ#(DmZ7D$d@U!CMgcOv!A~J%x@RBd0c}z$vT-6*9$oq%197{Ds1$+K*K%>e4P$8nI zn=55i5--)dnRw!sZx6k)-^V5KBz$pCn$%?j6;|Jp4$VoAFQlYSbkuRjjkI6DnxqO5 z4EMJcpBEV6UKeK>tX%^Ax{z)1R)16w_jpM{<}2X8sU#XQKU3n?q>*lwHVVYZe zaC`KD%}mmsXJ@X~naQ~AfC%Mm8tAiLNIFe20rm6v1lYM3%v z^DVrq(;5DwY?3Y_7p^ttUw73f1d^b%}an|#KRJ>rT<4@ zf9%k5?k$o@#PB!~uAz0v1tb&+a4}vV-9sKA~b^} z;=OF4rVck+mZ^~f>Ny@?Rd8pqF#n#!zji-qize6s{W+i8ucvB8w~vxfQ8Z`fI>Ek!L=K{%k)x#d3h9i$3jHS zp!O61@gv=H=$si(z`skN!>E+qk5_DK~o#EN!5AO?e4KD{uEdNP@_ z=6A^DY8Ehs5t4=>rC7BDH&)^kk6>RHhCupU`+*<2Mxh%q_IvI;&8TU2XLah7H?_o zypokAKICVlFZu>7T{abCsW;VA_l%4CCMpLicxFd4aw>Orr~PgFO1?Y0cYqvk!v^A- z@6BR6VuP7YL*+7V-@dA-f2?aiH3^jv2aa+1I4(lbvfHegFErg30w__cqV9x<|5mMx zvvz)DA1Gb%gNI`gR~iDw{%C9dW!j+@{|E7vE`sH1bXnyAI~9eoZV{9~$cC!Ipa$4; zPV~NFnl!%8U6^W%0EY-#rioOfj-LqsLTX} z+Xn2DG?u8w`oJUfO5V9fK}y5h&7!3|)%B+1FzJ$eoeTW;2%!@4{6)!{*;O?ifEi}u z%$0iIsBtF519v0mpE6GUG4*0R+HpR>ZEO)B)>=v0)c5$r?B{Zudsh6}XtKz~R2>8t z5ZI-?s(pdGqrhcn6?91|3@qcpTfHt~_O%!Aa2bn13&PKb?PpJqF0#Y8V?83czq=@~ z;26vUxyV+|a$*~>%rWsAVoBXd=qudn@zaV1*SL`*aBcXBfb3`>@j=;yOr;oz6TS7( z{gM0P>t|hdQrg$JVgq)ho;X*uXNnK)_CCw^wfyx6 z?ecK4+H&L$1!`xOLuGTwySG-;=arvzH03lY!E0{7M#-BG7e8z+I9T>0=hz1Y+@F&# z7ku_7x*TOGPvvqOp}zaMWX)6c0}<2L2PtF!o_)53rhf8%AKm8#wvR!VH`YC!lH-(3CoLfVQqa4OY_QBT7%b+;eq8W zp*WDP{_Y z+G>V?`bb4KXDX|9xARpAP8;wa;1HQodyw45J$0p+YatELin)6Ho|^-6kyr%8p(!Ykmi@Uio*c+eq%aV)ko2`<{%n(VH=M1h!0{=ciz;yaceD0By3WKY$esXj2jsrsN^ zz;}cEvZEAON<2c0nV4O>g4oQHuohCb;QkA7+Aj(d!y#+ERH>LV!KbD3%W z7qs=~%-oZ*55v8aOLoq>B=NFx3~>{_0jtH`0{zFZJ4_*bDK>v^k6wS2;5*0WqU^qV zXBfw2VnX5E0bsU=o~e#W97~wHQW3rorA3`!gKwBWpn@OcaoYZ{gTZiPzXCPmQ`waQ zrrtLT0Pn~w{Zl;aKKmj2YK)0sGzPHKf|0%ArvP!(^R-tB59jl`e%W+2ldP!xyOBNj zcY$@J%wGeW^ga?5mvmLS9^`UTV`@G{^~k&o#`LlN)BjtZY?Ety}#hvi;qC<3}Ux)*QnlcSxfmCkd_PQQ-HnL4uUYBBUm zHC0wB4V95yyU$GHkl`L3j=O6S^5>&Mg@#&hzIa&ai?F<#GsRO_eZ zdv?1_DwM3KJInnL#?<8YqRXZvCCLBuWk{jF^NVYJv(NRuVffGL6C~{z@0n8xRH=*Y zFdwZ8_)9mp=LH7Rx%3j1RV+nr0ZyhnG*fk2`I2h{&eX@AHa*u3!%~Tjt5N1+`{nwH zLIlaZkh(j?Bocjuj^V3XOwdETPD7rftszcoF$G-FkjebU;s*!5heZ{^P+8Ua9)Xbp zh$0_e&ypM3!97q5W>N9wJ92V+*u)FNN6?<3U<^oWjSguB>bzPdD z4AQ#0!V&LXt!)t*>E{$8sprF9Uj4 ziCP6hO=L~hoMx5F5*3Gb6*iXq(q$7~f5tVe^NmS1weQc>%aMAu015O>56>>~OBO38 z2GT2Y<9bS+;LqO+5cCCuA?ok-n+KYjXrTuU>?r;N?#{{RC%4 z`t!fZTCJxS8B`rPRGn)ZwyM(U$xf1LsjUtvQj?<)pJ~S+G~&u$H#n7;yN~EgoM-;J z;Nq(OU6sv#e{bBXno1Qf)VXKc8ww-$Pw^W#{>ypAmb@A-g*2}C$r0-@K4@=O&aZ|%_ z%QaGJ0XccW>+6oP6M&!tkli_#n+#WlQ=NW_>0GI^P}5A{pa$IVVv6l@0^=Zr=$?s0 zhvEBT`ftE3=vMm4#1oPLzBib{`jW#!T;wBo-5L}t#PI^El7Z6#l$G~X%|C{!X;kIq zN~1m)Dq^AI!#HO@^Vt%wk#Mme>p3$(fA5Se3U z)kaSU<1UOtZClRj&B~NA&di<=T@3o<#idNYa5gE5n&`vNO(Z%rtlteJ8Y*J+tl$q= zcsNBtX9tbbYMP!b)KNx%kQc=m^QMfh(?}R{;67f-xSZ4Zx>tcWP#>m`4rdy59uFN> zP8;yn@7JcdQlNS#%*+hye@SViq#z-fG`^0LZht}RItb6+Cl0gDd5<9P%=`NIuXgMS z>#*IDqA7ve_I0wr=@6xFX*GJ@V@bXjAHLZfLI4gi`Ytc~Wl0%QpgI6@Upo=(m*R&B zB_T*nC%HM6kPUL*d=cc;AGxA*k|zG%Y(0{q1BF}GKuTTZtA$HR{oKzIclzs@BwZ@b zUquh-9ePdPJpC@TL8nP)S@3!41uI&ofz@5>XGab^U{hv1rrUb8YNC<9VR%Y^qqk(8 zefNy@#HU=-CG?qGU6SeAaKX~bvzU+e^n$UUv5x=_W%{D=+XT8Wxw;2$GOts(ClTE@2A8I>FFv*&EI(xrSyXkk~!7@`;shB|)xWK5g~md3uZP6NxoO`bGd9eXUrQj+s;h zQ93MM(tpgY6{~dj&O(rPzM-}=qB8nm(=ZHe*XCir-|pbIG$zJjcJ8;)9oB+8g`%T|4y2(W0#XyOOz`WqoW6Ug@q(umU z@FLyezGwZhVIkPkLC`@~0b4S=L7@y|Q&?GDze<0~+ypRlKgz%Gew6(P)e`iL)yo2e z{i^u7RUs+1E%c89+JTLO6XmO2DOFar3^`BB=(jc);gf~0sHDH^hy|I#cH;0&M9qR0 z?so(l$RygBb=DXE8F{dbG<`lQCp0JXL1X>ZMyB4d_f}bWS4&%rFJ({$2N<`Ca@i>1 ze48On2Js5{PXx*1YRZd&g<*e0M#nbg6dktqb<=pXG0Q)VP$tdQD47p_&+Z--%=_CpLDRpu>C`x4+q@MtSBnlY&x^;$a-p%!- zoxGjo)wvk*WmuFAD0oHL)ZlF(pEc8V_i&s2j9aldCS38)`K?)8NNwIp_Vj5-sEbY= z!JilcS?01{jc74Y1m8+1(jPfAo|CTg%ut@Ec0AgwE?JQ6pL~=}XVn1FT2kdA61_b- zhm%daH*}m<&fgrV=?|P5r@am$G^*l%T5-nK9=+NB@?6UPh;R%3NhV=Z;f&U@-Ae>v z_FI1!5mB)c#2^0Fqodo@>@RMhuA|WPhO@>0B`4;mlCw>In*8)niHvk`c$i?K zMD2+^;lxVlZTx*BG1tlf>5ma{8uW@ifjR;wGk-4ely+qCD5NU?bPCUzp)SLx-H?h2 z#*g2c-ptBs25A!umzG9!v5xyA@pZ5#8~U17e2e*dMmTaqCM!-vFKzP^=;vP@Qja=L>xM>?t|{B%so7?k5->Uf8Ad%h!s%AV6WesBUqO27fHZH z!IF3Qz3)E)|5eAR6Z_t5!8Y`zTLg8?x>-z)KuTeXpzfgl9p1Uir|TGT{4X0d@WQd| z>7pmkVy5`MQt@I~qGa~HXTPnX4m*osTD75Q~aEaA;%SKBw=^tuD z9$RimHqOZ$JR!32jr}ecGxnF)`3yzftskLe(7CPt0H2Q9jX+g4k6Xr7qu|N#WnBuoB)#K| zhv=`WTete= zhl$gd0Ts=g5?swkdg~r&vx(E=l(&Wfw(kk2ju#94Yri~X{&=#ax#SVRs=YVa63>0n zz!xUn2EWrn>EW~c!C9?f@znmR>=7t?%OCZ>biouuxHI>|3O4hMuTSS@B|9v^;N;*lAGq%R@a<*M|2`x3c1R@w^XW^0d(q zkM%y=`6oN>IPde~?mLGPcJyCk-?fplckcP}gQ_&gD;t&WL&J4g2+99<^_E28*|z1MIbO zZsM0I{C#pL+pi3h;-nH}?wj1Dc%8aF8;0sGcB$lJgJflXctTT0OzE6WW>s2h-uqT} zSufFPP5xtUiiCFbB8)?wQ1I=FPVo77Q+&JS$`iWcLhfBU-6)S#DtMV#;T7)!oJ-0p zAu#mW>60UOxewyB5=D-3#ge0!{RlPzY3i?dxcwSOsR$Z;ee7t)N}XDTkHai z9wYdk-4{VQ@%!#1$9Nx2|K#%qX&4FrI+cxAixFi3sbzeNw>vj+PD2zB+$x$8vRnT2^&Z>fshu zlgX1o8l;UzkWGvRnW^Y_w^Cf$+rPD@Jttqt2$t>pqYyJ1!P+s{Se9(7`kSCb+*y2U z-+U>o{oIN>GBi~sT4rr_<}|Z?0dycz*yQN-NaiI1n-}U7d&I~SN!F!A@^^f^WO|sh zKlxLXKSlhxXPH51xe-IWiiRI5H6X4G(3Hwe{4VhjA{Da2q%cIV5td`FR- znQs&^so!D z;VOOBte?ZX83}dxjJ(Tpi5J!iiZaw+(RXMSw?d6c7mcRPv2VUUw?ZjpM13^j;~64P zoW-^<`Iwi=+?4b}Boupe`6ImY5p}Tr^^}C`5*qFca;8K0k!HeulV~?pE<7oY) z=Bcoy+=i=28gi)%85!dUP--Amj)MekuEs?g*`iXU1D4IJd$Dp+!Cc zC#Ti~|0#Dx2exb)SCpbM5|O6q;VpFaL$UEkp5eY#pRtUtk)jXNUjcnxq>j!X@lN5J zG3RX8#Vpp~Qgj!(xvZua(SDI_L*4kd*LpXH9~6F&2*?;o?+Jd8b$$YyIVU`TQ}yz+ z>gs;L zmUo@yM2#;yfBj|QTmRW9c@}Y1h6|EO)ztZkr?#}baR!FzX0-R|QwA>Y z?i`c){z;5zFORo7-G%wPhx8)mRKcm}&ItWJMviCQXD@o(wS?0zB4}<+;35j|2zF=sw;D}GPcj>yGfj% z&PzoEU)R|+Fh(0k_xm<32pof_kg zcV2WshiC#3p+&ZnxIjN>f2V`)L}C8pry?KPLt=bcoBF^devPf_vUWuEET~fXJl4Xz z@mi6TIRlEgFcwT7pSqimsuCPF*%tW!`Lb_w5q>&QoknZjh5RjBo5lK-LMBhE%_>5+WhlA|e>#gwDLl=y8)V?OK`U>70(ta)333BnehQAf8{h*2~%w@J^>nrwN(F(B6x1#>oa-4IptL5FZJ`pZs zgNe*yd<7a!qt00K2PT-A*@n7gCV7wjJXO@67S%Dbiof{5z$iM!UfXJ=IB^mwB=Yii z@`_&;#vHbpc)LlWD!A+RT7nM^*O3zP+1nd@vEP5v#A>-BlUw&8DQzZc%E^n3ngA80 zC5j`w^K+okFXkxIZ#rB7)+Tr*{u|QKYT`c)vo6K|fEzCV4~SlIk$yAluGJst6%x8S zs!TRS*!I%BhFLjD44y-lIEK!}ejh}+C|WxdNamjn9Jmu~w3VfNPE6WIp}XuUA@!7? zbc8-1ddyo9UeU)=>x&JBK%Dcw@)=Keln58sg@51e3ZOTX(_d|AP&mv97Byhj-$GLe zUTxRP);$V;bjmtSZsr*LvdOuTF?G%o8crJe8Y`Y`%pTC!nG&HJxBQA={bCh zZkk)su7p?mhjV(;7foGK1-zq~UypKI&nO*7= z58AAA)qW_-F{-_Joxt*hefmx}7b>unr+{O{DT6a1eHuFcT0Z(^<=&`$Q3?LyZrM*l z^wb|Ce5|;&W@iQQd?lsc{EkT6w-U}A?0XhhO5$XwD;H%IyG+iKu2T5YSw!%L^dWWe$?c_OSi#BoDld6Q*=-CT#-edN zmrUCyE%~fl7%aH}Uw;)e!Qwq?|Jd)DVX;+B@dI6MH?+IVt8}gE2I{4t_b{lssnDoD z9UYshl=?w~L&0?@h)ytiAr8wtL!_7cH$=Gf&UkX48?)Sh3nCpWjr zPhhITWfW~>*#F|FlnIiOD=?xzgBjrO#GmP*Sjke*SF9RXZ`5o%HYNSg6{Xe2=+TQ_ z9CZ<|r}0y#f)x<^=;gniM|Rz)2vuJJ`sPDpw^pc03yYcZQtycfvRuhw{0!QkaA@ZdP>BnE5>nR1lniFwZVQsX*Qhj4u zt6XG|E?PTRV&Mp`A;%!RrDK6Fdo~ou0y`|@WbGlmuaZsG$mJ?p)@$%JX%q(Dx|tZ8 z&JSO*Y;4dZ?c=#Su5^#3>aOJ2;ygH7u4tam|Ug8anF+rym6ZZ?-saUn^Ng)0Mxq>^qz z7DP+Bs_ACtVHg>d(?6c#*02W~i_RLFE+)^KL%)?(~hysVvqB-3wF z{Z?IiIq2fRCJyeceYf(s?W4$bO6hmP1-OrDu#GNivSV&lk2xu(fbvk;<3F@oOQhE3 zk#HfR1~76ooyoH(2G3L z+YU$xjrY&0_N}cyABZbca0~u+Y1V0uLHFDmPAlAb+);O7RmLrt6?GlD6mN6!Vd!n} zaC1vk(0UcJ3Wi|1sGz3x1aa3}#_~ui={G+|giyb?M<3nVF zl6o#}tzhTc@qC3}D}5>OwcleViaf3NST-MzQAn=|-nBrb=d2 zf8R1;`etJhIGACa(;4^eTX_shO<&fl;orEI`AniH3VCk76;4eC#OEwduTK zE=&0#)fO0oBYunMBkoR5qODiPOjb0PuX6ngdl?DZM?ybKt3_W1=#V*b8OsI?9U+gl z=y61i+%cUBT)sTk+$5>)83L> zYC??fN*R}u$2?!cTnymvLiCgUJBLM6zuthhRT33P(YK|?RXjsUdqE*de(%cXYm^I< zO5G80XE>t!B7EI+_7}roF8A&iC5|Hgk2Sky?X3edCMqe`<{}=gy33P{1vP4S+ccyPB^<>wX%TgD}qD?!SGW;01>{gnx-Q-^=T_4bz z3haz1^b@`m+=bhcY2zBeL?hf5xDnWv|CC*QLoGIuC!p*Ma#D5?{9@x16?VP;uX^Rap}t4TRgIcbnRcAyI5RZA zO1aY(q&~M@PY5ksYrYFX{ls=&;MSt*CtX*tyi*FVQ0%pu8!oD2H_zwZBu6QVKw^aH zTYvMSdyg4pLcNUWx)v7>V3v*0!YQiOSrri7UflNLmt|R1g?b3JmJ)hoRv2K?G?dxX z(^bpQ)=-JtYM8nf!u4HF8Dm`(py*C@a#&^Gpyz@&*As6LXj%auy67lb z^vkO);T4zPBaBtSD0xEE7^Z2Ab3BfSVbS0?9Gp>!1E~DIeUf9@uL|FiDqyoJ+D~QL z54omRwkfQKF)0ikK7R+raJ!{Si?mpx-O{^0ZsccTIy`lGcKzUTg%IP7CLdwd6JOk?p1A`Ct!9pMRF~QZlnfzFmg( zd$yXuqHuR>o5ID^^y3G|- z{lY*{{H}pKjf*4Hwav7NTGZ>MHvSQVM{l}H+QdmN?|rxSUDhpY>(dpJ;4xL2q;P1M z5~L~=L%om2y59{kqQ{P4L1f!H7zyGbk5@VY=s|h4k=jVO#ID(RA+UA$hkUrC?wP3+ zP0e!rJwOEda)uW+mx%0!E-McO>8LDy6d#u&)3(1PbT9Fegn#?(;3gtC?tESBHqzY6 zYhHN=o!>vkJ}-mKep7;##a zZy}$TCiXAf%{phspK4y1n5u0-B9P4yPCL~WFB}{c$T*q5XPIfe7=$S~hGuSlQj&Tm0RwbfN>m25L&w=0*Q4Cg$v z8etp;;U~JDUY-`_+Ie<6bncOm`#N|6iFVJ&=;l}`c9?cFnEct~I08>-XRy)e-HdaJlGfyUl?4+N1h@h=#&7sL`=TM`)~fgJw(7 zJh(R)6$;5)+zvDq*U}P4(VCR-$WUeW<~JeUHfMG!tYi0D8cayL|0pux&2M}L27!t{ z=bj~eiOs|ex<7uEYlFp-iK~TPTBLz4dK@86cQ5LkpkUqEp+VSF{VJbb+;9%h`Dh15 z$sXJ#m{q8O)}_i)w4xfnJvF`iIUkIdZDHz;sbZ{8RrE`M{{6B}2Oy@yQY6Ddh0l`7 z@Yee{Mx3r#)a#$VOE7tQzVMtUPqJ1FFYoj9fj5O#I(D>Kb4i}dQ(x@~$FXvpO2)j%$*fC!`?jQ46m@f$UM;{XD;-uz;U*)lvp;O~We&{s{K@rm z*v6Ci`r*djYcNvvNzQd>jLOjr`Ax2M_wg1gVuT;GJ#u#tPuc#(^MWzID5|@-v<_Mr z#c17SNrweo-c|mIEa|sgLiS#I%Sp0TazHgn;rd0K?pU;#;K&f&vAg1+9ACz-s>D04 zk*F<=F8xr*BRj{{3ul+$xUdV%X9!Zezu2un-Le}jpXs_l@-OgP3=JFfo+dlTU3-k5 zNH0$egpF8!_$H2Z zGY2LQ`s-Gg@G^(US)$DeaQFwm+6^ux*0Qpo>b|}FiMMP1z+@utI$;8FplvA@;jgZbXc`%bsg(V%9LwP zDkBgn)l#YU02jRh{7$58L$AxV#g2$AjWT(59zUY>&62rSnv`uN@=<(oH^)1I)8B{r zrx1TL%A8@^Q+KmKkVxbdbwj%KS@ptq@?z9aye9r9=Zzx7Mtxe7W4CX<)WZuYcPkg2 zXr2piWSXQjfnjd-7caA|b2=aflNiI_B> zC}?tuYlK7xdIdNS>;&AMzP-sqTEPY5T3BYMTI+_dXhx;%uVxY@2rScn{&`FN;YnB} z<#)VWg}uNQeF{N0q6{rM3q|)emnE)4zI4kr-}g#eb2r`A1i zvR~af;zp@mv8Q%BiQNU4OdDD)UXzJx?K}k=pFT?2_p(a2%Sz1|8;UkQK!-V{jGK49 zm}fZ-d;h9WepGP){;MC50auPzihf_>z2j$o5kMp10TC2HNfFZ9_vc8Ehj@S5o* z&({cpoC^{CZKxUAR$6D@1bKb;g{bEIa6VCn+3-yOcrUE=jru5ZpM%>zK zuO|hpa6hsSbYmN|s@#zZ-}`qj5=GF~_0y&dD=;y~H;R!t3)odwf7mU*buEmMlZ@eF zj&nxI5e|x-9YklrmV|y>ikT92pI|P-EP!z$X%tJo=Z?i>-OIg(5n4y*{IKHvT91D? zrS=O#QhX80vI%nV?>6-MZr%y0)WV_A#zd-H;!ikV@@DzFtjdrdHDoRB#3--RI-)jb z@v1fmue!Yo84r|p-AGK;zU3#9D3A3G;VUVHb{te4ZjbC>8-chWWjH*3vHbz{r%n1Paxv1o7iy>;ZFpypxG93L9^ImSbI z?h3}fBZ?|(7H_l`N-*CvYS)$(TqFLi6 z2+Rc9c)1=KO2V!5H^{+FAes-Ak-^h?x%q?Z0iGuTN6N1(+~jGs;`?!}7tE-!UO3x+M0P%i7JX9c+l02sX)Mp`_&d=iE_Hb- zU6W!btXX!^M#C}L_3*s@k*JFOC0ULbY!%+3Ftfje~1h zTN><)Gt^M8lXOWVvw!L>N_(L4qSV0%%wHY+8ES5VPjR7vZN9r`O1?>amwU-tI)=%dij{8bN(+{e96Wi9G=JMG zN18|yRXppg~n z)u6b_!q%+ek0eB_q|Vm7Z{+3~%iv8vXO)K9!cl$8c3Tr!JaJ;y6CC&FN6REb$L<9ade$vd z3W<>nnPqj(yvLPpoWelEjG}?8D{4xTF&74-5Rlc;AwMIKIRzT(^3KO?kDpInujYOLZTUN6{I^A|TOjPF^LggY|s)-|Z zq4_dsgO-dJD_k~J^7QJf9FlG3>L_#L5qA?DFNnddKks9HrO{_&_qw62o}y#kUmRJ1 zp+yfOI{yST*52piGLVJ^;aklmC?>c5u{mZ**D!s*Wp^3hYV7wC_j5Vt=k*GY;TX)l zu#vI{$u#YvmtB^`y`wyfR`rbG7|h?AccBEm!RB{)G68KnzY_jbEQw#>7(`J~)b-wJ zF1{}POL}tC%hSArbP0b@Xo`I_&KarGXl~G8&?+ZpK9~Lo}0rGwk*O><7>OeQRjT*A@Hc+k5d#&`(%Inc~6ZX ztb7D5I*-WW`%fOWYc?gIK1ynSq8G&TE%j4T9NFGGh$=K)>)Q0Rf*}3dSMh_D)>s*p zPfqF%B;)U{-&f@D#5j}o(3L#A^Dsb7Uf;*Rj~B1PJa-y@>EzGgaG4?8;VFFo70}^u zDTJJz*M)9trFhJ3iBf`VvNBNcUWUd=PX%(- zjvBIZaotiUZ|-C#+wm}1lcHc0EjR%q+h9AFa_10q*IM62=hbo;lyIP1jjOF)E z(kzkN2Az{Ll}KUFvqt4M9S?`_7!o&yP*w|^i^T*Y*@kQ+wyUa%opYZ2RV~x6JeS+i zg{r&%QOR?9T!($3VKbH{J>5fmwFS}ktJ-#E5Ef*F$pRfvS)lj+lP%zeelRzund2Z+ zFv>eH2fYl*mjXiOygIo&Jsecn_BN;}>YVKx+N$Jwg`=B>)Fbtadq?->VR@u@oG8Y% zLbA0o%CXQ_929dE7|yYwCXcP_b1!B2tlb=Q9?ul16RY!_Qrn7uB%UoFtXyB;+J6%_ zeJkVYHY>rNClV1`&-I+!-l{04U4AJ2hm9NF{1_Sx!k2H0$FF5L|<)-DAh{-#juv^^H zU^V=H`{UaG0`)wo0cNUdou<-9w>y@kWKtrM_0ZOS!LHxp*IaLI8_unLrQXyV)2Wb} zr7IK3k4cg=v)Xi@E+})p{Wkp#oA9~yY}=)nEm-G>STs&+LdWA{Eta3Se2scy!unz? zDg*zHO>)`!gd>LQJI>`+3-_$4*hxcg`SPm%dac}L+$VKNvtF18H0YYDX#%($1d@(WzrFJ*i+s+c;&t>+b1P{WLVCTzP7x3#XR zC2DZ7Ni~$D_|8{aXZ*`VM{P5=MbzSSjT7CxrD)93!8f>hI=^2xgdMWbYE$>s>2pRAQsZ#|xqu%M! zGRi4%c4)7*aL#1~2tYFiiLkS-c?5}mJ$Yk-6B#ikPg-HD|LJZHrcSEbZDi}4@9E0v zwpz|N7qT6r$}(Ef>d9+b3xDx^eYR(;je(J7Qo7d+I@#dxggOHcO{)q}blF+)jQ1MvLMge0u`y?j`Q*sm;ZAP*ai=$n1R=|^+fL6-p z6ex~YM7|Fj$=HRAe#(=Y8s$`PiNM!V8fQ}|yyp&B`y>BEYV%p`TfFo{Xa$Nh>Jrv& zM5|_uixq*{?^2vtNc<~n_cq(dNuQF>3%EKTkNfP@D}AKpD%OU~UdAOKMc#?t@>ttg z;{=wmWMp%Mh8TaoO((Cr*&V}M;7+U(jR2qWCpDt2dh zeOWi&a8ZZz?_u(pXx7Waot~N@eRJ|wV}+;8tbwC?R?U<;T88A zKA+0-p`gc^S2n*QF8wm}=S0(*gmY}a8aDDVA7w!z4dswHmS>}bwJW-g-Ljw7P}fHy zxfa&d^YwEZA=kZina@q-qb!(Fwbq2Ut+Z;|4UflE2zE$MUAtTscoW4~-BAnd#ILhf zQ=!bd(tY3?P$SyeyEN6|$Fcos>M>}8oa;@l$Mqn?j*9)h=qOlz=oSo0*9y9hxzeH0 z{(31oBboOruU?3c$Nm%vg~pY7R=XK$q46_q?@uAaH_0qs>D zj+xt_!QD%mQbB>6jD*IUK;@QS#+7F9!qoFxgp-Wjc$+|$;Rr|?VG-dA_83=ukc!U1 zdgk=o>U{)mvCR_JJ^2#V0+Rxz@FMM`+{RFbPU+l|LygETAu}-UjA6$2q&UlG){mmb za!ivz`Lz`LIr%>FD1?i=Q#L3r)Dc!Pfu*Kr95kvJY4Ukb`hZgqHrJOs_DIM@pO)jY zK=BW3!;`N%vI$B<`;8V=a!R<}YFOxv-G~@rPqw0Ohv|W?ustn5eTuKh57V_uMrL%m zasIW+vHdYdO0c;fz0#o+Sep+s?@7I~PyWV5ywD)uGW3OZm7DH})=ko2~aoh1Yu{rea4_)((P_fMqe(Zy1*MYWkuee;HHa zwe?zkzw{AlPS3V34aRCy-%Tl3J-kJgSK!3?bi=r*P_Tw1F7phN~paV)9$FFgznB49Yz7A39W^Z1{JU}Sh&wJR5U*XtZqKQqJ zH9q3?vYNWK;n!dy&1jO1r5J)m8E^{T)52(94Hbcj@ z6A3}_=4M{XcBR7%q=mafKgz>ZjkDO47oRHKJt_@j#V{?pwx0}o#Wej&IEHSkgo%qY zJ5#%qm+whd7`W80CS3^93VxzV`*-ihlJI#`zTmXy#|s=QF~~lpKW`-ENGM;e-O$EW z`=d0e5ymQEsQHXArwkO!)A1s)jZhIwKihGpisJM`gP)JEeP*8<>!g+6l~hRB0U$H zaf7KLjq5gqwp@^@PSqoc_iHqcH)~`Qc@$16-#RN#J%(aQ)aaPuuh?oULZDXCDJ1r{ zC=1bS+`Q+bEmCTM*N?{agScL|$^IxUp?4tZSPeGJ4>v05Q2VAUNk@wo69IMN1XGmF z0Vt{+rfR;a2J+djdJsRR2kvndYV2h%fHs%v!ks1}#>w6f(p|8aTm^^}RFEX*7W z+8!$tH0w7M>WmIl@u^{)6CZML*0kY7vnbCihX^vT*1YAD3Is{x350(m$Uw6Em=9n5 z9VyE15*W;Y7@E}3xvfyVr*GfT=XBpe?Oc<`GRadLpU_0QCx}EE5pYMN!p262m(#}P z3Q z4O}L@FXyiZT&$3DDr}cJJa>DUm~Tlc$WI=V#+7%m^#Z8$Re_G1Yy?qQbMA*%9PP}q z+trR@9)AEcV&1Ac^*XZw_buTV7DE_Ki@wGvhhcHURLAwE6DE-^!c6)vv2k@25M zrx;n&p&~l&zqC8~cSO?|myp@n{l^I({zkgC$ZYgAP`5w6V*K1oYks6P*i%pD4+h3lrtQMu{_>mg^JRBe(s|;~#ClO<} zofx`X{S(!Zu6}&fnYxCno34C6YIQHo*i8~P9>3DSOhSe~|01`NcE`BD?6FiJHyyl_ zUrNnh{_h6KQbMW}TYYGZf3jSqeGI{(L&Ks4MG|Byzb*z8r|&J8F|}!jmTPDN? z1e*##E3BfWx^&-f3)wN0qOoUS^j3X8avPiBI>69OYNyEiE)W zXjKjKSAk-I%~|8V-xGi`#B0c=RjvbMfT1LUpqHgH-`tN0A9fgZ8t`}I0^i|-M!pIi zoV~*HUTW>5vy^40(^i4S)(t@iD2FsD+~vBwSpr|NC^CtJ2YKpALmopV2sUWSRE3%% zG*EsZz%VL3i!xtu$X}We zVn5XTBd3S<$;Is>IF^#o*%mgaZhGL1nzV#VGMYgc*55G+Nl8hYlqLmiVG{DIHmEQ9 zi;jv&dd5bU2=948QHyCQadTn1C?Ezp)e{%aN`L~Rl|Z2fI#oJsfQ#OlJeDnFsQ1># zQzQ~TW)9iwc?O`j0TO6@bPi0%fBs_uU@BG_UzTrTm*e!1AVYLiuO9hIZMiU{U_M$_ z{da+7A%cV`b$xPWCdB#n3eAp%Cgx@wetg2u|V2-x35z&P-Up^L0l!X z3IkNjm@Gmkn=$6YD_lY}*d`Ca;qCPh0vJc?TZP{m^&Y@?2~)U0IAlF zUXDRn@ZvMCvGKd1m~i2s3#%IdEnMdI7l*BiffA~?MdLNB(zxIq%T{ULgHFao!WZEy z`07}dXtf3v_J$LX62v~~8nvKscGUXT(6eRbc^u_fmSsi2&YYWr)p!Ry0kQH21d)`i z!|JUUfTwa&8wbY977;`cYn6=Y(c4Z15Y=_dQY(}3NPeCe!w#;k-)8IlEp(zkn>vf) z7MLR!?9YP&J$yeJ`*q&K#4IGsJw{Nzg5%3raW{s!n)5@}>8RE9F%)H6FZ-Q_1*E9l4 z--+qpW?PTMJ>a0@ETjP!Y(*%|Q^?Pe_bGFX4b8n3F%r?&AP%}{Gr~}JfczAWKM{IE zj_Qr{!UnE_g=t5sf@aKWGqjq35j*@MgsSWTss3z~c6AWn+el5SWfv|Id=6IQuidLe zEzD1`IhEja)c%o^Q6vRYQhWAq6(62vQLe_(hVqgQ9Oal@agWm)NdgwO+<}Cu5r_fAGP^)-ZXHLAJ5h`A!{+G>ulxT8u9Bt8#(G**3QmXn9IYSV!f4LUG88)1mMga4gw+QCn^P26Iu(KjdzZLVR`fhOBiOXapBOj+ zenepwM`Wx9A0-8~i4Zc?Zjs2F(l%G`MK@E|=0>HWn}YiYpErjy@VChfKilaBJ=E-! zkp-eEjG^=vLu6Jv1#gIrYWb?(l|Xa7By7&vkWtQhL*Y1 z9zFQ5JYCucfkFvdlh10#6&OAsZ#MndOm_hP0l2wjb*5Z>*GrL|8A`p)EzIey65n87 zH1764)|uXORczDx;W{2kOqh4b+!*OnQAk^JKH%R#ck2 z?mdh(5ZOWQ75QBIERsiDH(hq|sv}qG)b`Af2?eAh34lb$W zuawC`_y)AP-?GaH#S)`b5?KBAG_>Kmekm8d1rA#xV&+caHFIBollLP_OS=EL11^FO z5^-a`obZ_tJ+9{-_pOR5F89I>YPcp9iX%m-QhD~ZSQTmt|NZ%Ufa)KA@= zsm@8?UT!5v5IDgM5XC)afINI!l?HqH1+q9n_V*$Ar_4R(@H*k?MhlhybEV`cDFw!v z;@}gaTOOBZYacmNBhta*_8M2!w-JhOuBSN~>+&5uEI2A=WB&=Tfd3@`FjKOhyMeo5 zn)+XEACi=zk!ApUduH|;?KBdJwAJ{vY=~M$8j?aKpf}ZJVmG%gT1_!O;wBV%Z)#XSAYxbuckQJZcInKjHM2JrJpiw4?G*GTmBv9Qy|fuX^zia2=={XomoRi>guQ%D z5Ft26X7|D(z#C|6xE+?Qwz^5y!jB(0Y8HyVcaqBGqjGs{VFm|LQ2px_}7ghPtF5LT;)Hu4{mMd3vo3fG` z-Te38Adzlh4|D@&>NZdwe#5ll(5!3`kVoOWWGlam2ZerLplv1&TC z6nSR_yJ)};zbwhg8==vjMe>js(QElQtA`33^AV!71hgmjcyhJDZO0dsjw;qn3a~?@QSu+ z7>lEGrneF~??4+_`Ne;0x&j5ifXits4k{hoPbGPv5LPIot6**g~jBf)?Rmla?A}Tu|(`9;Y}(C zZ3u*Z4VNWK9mSiyHh}r;LU82oVp^smFV>q|Va?Fh`Q+yC9H__R*Dw zYauKP3+3_3?o;AWiEasId!NZ7X`I#w+dY%YYkuP#pH-J)Zy0XwSQX>n-0inZLiZ zJJU{=@Axk&QT-j@H>6ZWt~QO`V{TYz&y>6Az$6asCH|bs|pT1Hd%nc^Kzd z1ehR@_#$?X4F36^e@g+<5Ubb!#4zc1-iKdz`0M^(*dKREOOMc_!Sg)_DI_kM7jh7= z$)ij{IyU@R#2$g7P(eF^jzCm$AwuP0#Q&Z(Z4glP_5tGj1W)t>QoK_9|H1@zWvfXz zUD^%%(i=zZ=XMG#fBg!#fT$ z)IUBu`QI<^2fgWpc8qn8RX~yhR;js$k&%M@dQKo7FI#l&5Iok#TBWaRfSZ@GsKYsep2Mif6pjws*gv{QV4qVlgLSLf-|Md>p z&)$R&C0+7Lg|-ooQgji4DkrHbPpFSYI{D3-KtEgZ>~^`N-SB|{1!9BzyKtav&8V5j zKL)}}bq~;0TO&%PNDMx$8b2%bAJr5DQ4oM#t6C0Nt!`*9E+=S2D*tM2A@%>*a1Dlg z?ZvgX{qGUQ>5-130}lakPGV=Vu?%sP5f{&jysbktr)(3da}u;V3$VOKn}&S=A|NF_ z-{vK#&Pc+4yeScLX3c(3fMS3QED)ikAOiYWIzi6Y1+ZTOSk>%_E({mH980>?l~cR` zjn;)0&h=)N|J^+77&s7KDzF?YBtWd-_ga$fh>czk!>TI0)Mi$!H)L zLBh=Bru71$bq|QUjJP+uBTS5!o&=@J$zKjA(13I0#AVT-|U|0;dX(vXhV;TUFVW}2EBG@+V=3{|p zHwe8qfdWB@bwm8s3GhD%y_S6kAo!b8Gr<0jkNcQ1K-fflkBIIpf$kZr?i1Ty_B;|4-EM6t^dZwDwyztlk?I0k*^-j+bb$wchQv-|I9sjB}kMp&eV{61un@h%DO zxIj3;bgbGOAw>H@zTDM8Ad2wt=5Ae<(+D4QuU#a*hI%yfWO4D%LDEIQwXuZ2FrnP zgW1Zk%(Bx3(E}XtiT8~&w7)+q%U~%~0O;uU94IeXq%}hRGlCddU;0HLR@Jog8;rvV zIX=+;5{K9kan4EOJd@ybqahW|>lE&Od5LWM4e+jUHe5Joxj`JjRo|6fOU!(V&Dj(Y z-ImY1<>>PlJZKj}AC3zY_tI4Dr49qmpR%5*FI_Y$bFgXy;{dF7h$>JbDvPpQS=WSEn_`x-3px9zoM z+wyO~w%-C@^nr}0*k_t8Z7$6Ljvzw2gYEjOtVIm@57}2u(LQ z2H$Yv2oWrRkavdQjq;lM?C?` zrl6v>ZSFfqBK)1a>&rh1dq4=~#o$KD4CbW+fbK&)%$mj@t2A-*KD~Cj0rANLfp#GC zRf9;zP^TPMO&qz1>q{HuVF0hw7xg`=^QDH)9SGzQ@YayZ6g+rcI*OVOU@!NR+GW0LI|0p^3cD; zI4B25SO$1OfwPVHmzJ3=OqOCYxYcYY5b-l0$lhRTJ4lo01__p+T|^w=VKmSUD-C$W zeiQ`}bfsZ8E+y@Gn|6w7!WWKj%XN49zn;Hf*aundi$jct-C-4&tW(#p| zhJ*9x-S+7!h`P=YF{^A@FbK$!RzH9&2ixQ05=yh2Vf`hq$4tZbvzhT7-}Jk!l19Ko z_CV;{CHH1Caet3e@B_GQgxJpp;*^x@3Mh>w#X6$}Q(V3xGn{WK3}npjfNtfm`NeZ( zYE|55vr`b^P5>n$`tRYP&o4*CJ{3L|(l}`K=l;J#T95_>p7|)AnA}m&GF5;7DIM5F zeQ?j=&^)-+D#_`L5=pBAtrkfp2*c-r!3WEYFg!W~Qw#neV=!DdSKk>SArEguD4HGC zYYe=%VzzAE1Uj1?HD;W*pwdlL$J}IXtU}9dt%)23C3qV^8*JxA?jb5`(y1Q&o63n|EwbtzT}1M&xU@$hYOZm z^oaf118i^APB-sd+;ld@dUC+z5V#J1L=q$Jw;`Cax2o#$v1n%4{kfhD!m(CxM+|3~hzEpKI zZY`JNXZp{mI4}vpSx}P>9QeBleYRCRa44{L%3f4|x~2FQU*EpLZrr9_%jKE{A|g++ zK-;`e{EZs$k2(2#l7`wqUT3eVI5Bq6<_h?{I8R2>>&z?QQ(~Hb`~{LtOrBjpBZg!4 z4OUm$4B?A&5iG(N=02({W2PLFzp0ZggUX=i1&23<{t41#=D0&w(455e z4=^vb=vR7l)=>$r1j3|r&};_O`_XIa#_h*xA3h#|-8ip({l*GqFF}z-ca`?DxQ4Vb3e_62+R z1Z3*Z-Nv5*8whysAM2Nyu0M`xOSW~;(7L4dOYtjz**htSE!FtrNziCSAvl&y;;X>eKsVtgg2;eN5ayY1su`Szfh@=aJPG9NQspNQ81uuR^pZ9Brek9v|FSmhwQDtp zpCKB%GtHYFgg&8QTgXZx5ldX^C|Ei=d-u~N~AHvJ|1JeJ}?MdT#lO4FY zOHCOBwJuaTOzfv)broyfcpn`E=P`~{>b}Xc+uU~%i~anzAnjXw=T%orj~FaAc1&4L zS+E56pt4am=3&*AciXdORQ!38!9J4|f%CC3-!)cMntN%~b2FXv@cQ=u5%!gFQElzpfP)TQGIUC(AgMIcDH1~|-5mlF(p}OapaKdC$NZO>89^L}|h9M12|nb~`MsAGBwFe7vyk-nnQ8*#T(+36CyMx$pGzG zFVsJ-NI{J|H*qFR#^6NzAWgN75-YuiXzOYko9`PSPah8NpiE`<4WfFHmnIA(E-P6sR7}g3rn-;fR_eTx32Fh+MMKH5DzC_6=TJU}t z6aVaQzuLY3k?!6E6Pj8LuUGh6AZTb1Ii(U24xHXWlxFtj4lK0AHPrs+tN^6_G}M3( zb)IwivYmT-fBgmk+*7VF33O9FMc{HEv*dn!a^&}Mtj7xg43gBt?}doD}?Irf6& zyOMb)Y}sV&^dV4ff6kij&6}V2+XV=hS-VpCBge7+=v2z9PLbb;fn>DuH?}_>h|J_H zB%l|{RR)nom68(l->W}O>8JcC4YwFhpR2-{7kKTq}N%4!0 z%5e9gNZT-6bNMe&ZBuNO!4Gnao2fCShSYG#kZkz?rMDf8&66r;#lD#;jfhuw8*PS> zFI+$1WZ%DLUY&sL7aqXGQ2te#QN{c-Db@$D5J+ntLP|NY0!N+F=O3EylN~*^$LGW! zcZRw{gsl*?apdAdS*Vc9={%W1c$iQy)Z_#IPGBQ@dG|PFhs4f6jub8sw2+H0eoWDL zsGz}fTn6%_=Ct8@7}GG}uwcA4DUme|sBQE?_IQ;hE)7;Z8A_tP1Dkr3t+>p{2wkRB z(ZD4YZ=J=PA;C7N3@|t2i0xu8Z>PS$jV^Y6OON%TYS*Bw1I29WQ2BhD#L!`Ga}6AFX437E@FV;Tf+kuY=QsK4NFKKAT#?H8+TXPvDD%MB z6eLLR()aGAHVIZbGw>pcyIz}1u(5-f&i|65Yyws*2zaPOrZ|Ymt|9I~l$n%v%Vz4A z^4lzwEgv9aG6Faek2#ab84x8J7S5@V-w5)7S_5Q`JnE%aGUBuFJB>J?bPOHa4?O#+ zPO_WoOC*Rz;+BrfWcQ`;k#>SmjPEDCsQ#m$I(SCwhD~#LLcLVrG5TQcfiz2k4cYnE zPbET3_!&|I;;Xa`m)Z89)@d3dYP;EAehp=)4SxEo(0C_Fm?{^2eMa9)xsK=U(?HxK zJsndRNGD~>442ljIrC+Dp2~)5;_AyntpY1!g0WddzEG{DU5V$UIswVySSA7ySIP%W zfXn&?;n|K75zQF_hljbif?aWOiq zh;-tPbtKth7TdIC-uMazhjNMtRvpJzTq5M#v1NvBDm`oftxL-l^1pMj2pz+XRl3D5wP|2ae`*6@LEUFbg_aw+r#rT8MKjH?0-OFkQQ)_arA$1};sri9k~Vm>roF18oB$~dh|I?jYEFPr-B`?m$Cmu$`$6K> zkj@Cx6w1gc>41j9JTf))z1UZ4CjXRlOe%$Hd-)pD_US3^%z^CU@BvcyWF2gb@SAkQ z8i{Hz@(cter`w*8$8*2#PGygZID%Ry$tH)wov9mFLOH1#hg>_{kR&0eP)plL6jdjZ zQD{Mmf?#FB!zoHi-TGCVF{b@L%&XlQUiF1!;jL|!DtNBSJ;)KV*I^74RI+*BXJhw7V_~JDT$LYo z9m<8;s(Tf?lb9tCDkzu`@dFMv!7c4EHBK_ViImXcW!HBR$EK2ga06>pn`372SP0W* z7PsTDFX;Rj=4dm3(g}b46j}My6%n?6-SLHuUiVHfDaDTVTSeo6oG&0%3^jB8qRldv zdx?mCy18C*t@#<+VVQ50lDn6Ucgy$`L?Hag4r_T(llW~3HAGnGqxNqHAYn2om9DSh z2<=+JA+LqWBu8_jBHXY9IPIn0%nB>UErDjl@OEV*94>tk^$!*npeZd)4x38o!*<)$ z*lzaJYb;;cX?tbV*#yoU$*hH(p)>!Ua;k=)Nl72j6KFP#wtq6u40T_xS-(z8bYs0z z=3;f+QmfSjte-v(&%VrOYjy!)EnI7dx|rOq2DSoNuM#p%ZkX-6S#Ky^ZnTO3LE)13 z83OCqK}G>5<@vaXbz4UYt2K& zl8Wrl#6y6Hi={6W`Ng6z&8L>rh^2Kzi%|OX`*$hj z*|-OpC-Ii$qhoLRZExu$g)eINe za8%-3QIU7mE0@Cc*uqhKE_GCR*;k*Hg%kH3LKP8$?#i!re#rWddEW5hbC6Q`R++{K zDt5ohql~c03z%sls51Mt)Qww?qg4nf_l>7(ETqxJ=EPS90yOgw)Wa84xtYQm={n1n zn3s2mLxmaOk{^>4Mtuw8V%Tz(d7?Y1ZxY*0)X2MKi-2?Bv@HuDI%Ru*fXwNV?VU&< z4P5CF^4ohq@>j>`*9$hax|S}6dOTy<#!zuv#FP7yKwSG>7`KL#8{;xkxhEE9x#%q% z?;}z@I-;LMBwoEzQy3~knzyqN`%-83wtV4$7NeaHjFGBxBKmcKk+GG%aG8QK*DHlj zr?w{MEE{5oif&|=R>}=VH>a8picB1q6vl0dpC`rno@{Pg=Bo@!t`1dLPh%fp35wvI z6e9*uCZP>2_9M>>Q?QdH*cGQOpA@(wtngteqoB62IuZXK)L&YJ*~W+&U$8oTzq(@4 zD#1zI#e}|m#S%$d)8@Cy|7G+8%iYqcP7h3|XDjZe{6GaE849fhzi;a_Q( zl1xOhv~K{;*=ghr9Jqti3&!|N_Zt+cDU~Imxwt|N_GDxo!=UbFV#hCHMR~1?Q%9K* zJ+GLQ@K7iMOT>=y3gGwYrQubsr(FL29(a-Rv#-C&$NO>K*p0CI^flEC_!2)Q8VwPd z+n~yqG-dxhPp7~Due0M2k$A~KC`@#GXZKsj2GwhYU-Q*DED-`V#F zR``GPEiPtDrCmPW=$KvP3S`XS8T5Voq01j?joY$@KvC{~%BK`(JdVVEXeml0c`xgh@a z+Rt}M5dy1FF~A1&=B^B)Od@d;j-(O2!b6g@&9+u3Qc+oh+-P=EE~$N+z~H&RgEt({c?VWlb$X;iSi*&xj%Jp-K zWpvWc#^w@5;(tsfN-p^N|VE}KTNcSdi|Xi^++!OX-3bdi-R_J^NEZ9K_Oo%O8BT_W$U(a&oXXVoCX8}dkT0ySU+m>< z=rc*n@`N+7YivKY`DSPOZXMEhwy|^+Wga;OB!P~}#s?rLw)7G+o1y*o<`cJvH33$b zmc!TQ0ys)<1+@){qOzu+1u~XLhv12a&b0uSUErUwMRMo71 zFnb_sjDU~hF4WbLsq7q-Btaku#W?**7;Dcs5(OAC;UPR=P-)ojMB;i|4#@8q9|ku1 z76{E5xpJiFha=y12pb8QGbi2<4v)}b)9Q}7i#oJ(|Gd}=TDBUN!;U;sGy}XP8|Fg@8LymPN*y~&Ri>NH-%CA zLNKGbf8c2x&!WqA+@SypMk-SASlulS;|R7emGnBhi1;F&0)T+I@90l>UFA;USs|hk z;nA8%ydY1%5X3{gTlvK@r+@pM$AG&}L@`SdcSR6CdOpZ*ltA{y@|BIdH`2> zd8mN1nd`07r7^9AyYS&yL>ARtHYlTL*?LLt>|^O|Ugh&FMUJOuU;GunuwM;K6>`&S zj!a0n;kRkI_6RAjKqkc|_(3u7ERq=Uc5w}PkJ%|lcQ}esWLu)LIpcCiQ{R>R9doHi zE;DIYoz~3)`_ecnE#9E!qO0#cc_8y+UKuG8cmAWcG2*&$f!c~*_ipR%;Pj=?dB7!E zbISu1sqdj;Z8nQ?c)XWa^oz8nhAKpeR5F@_b_beO!|F2QR+%L%r`5=m>$feC68y{o ztQ)Pggm~<57V&R!R}ZcO_(^M|9eV!fEgzm4YRnhZ&bj+_l~vxI0$(*Bi3@mY^9TzPY(_~oaTw)_@AxQ!XM}w%g4<$+0BX#NZa=)xs~C@ol5zSh z{oZohHw{JZUgzqUiecr`QDV55qN!6FymR##6z+vuBRdx3fud-#cd5Z0Ye`}K0akp+ zi^(CMSuEM{d#Tm^&m7T{ua^Ww zYI`Mx#3nv+8dOSz-*bA~OxkJfp`4>W!NJhNEJ|aiVtxxFP7O7d6k(1PU*LtfkIJ_K z#9QY#hC`7|ChNxpo^5DndBzK$Z0?HlsZ?3W%{@-szA1T`<+!tr?oo3m%z~1J7@pG! zoUykv|6p8)e!%g+Z-SJcAS*J~&i`sX9L#zIW?VTlX{;;1%L{emoPJD^W#zWp?=EH zJ;u?;s9t0#Hx^SHoS5g7qG~oxeME%Lq6*8LgT`7Yq!KeGT!4srGM?NAh^@Fa=msGt z!-K7Jnc`34C9J_Pbm4EF zZ?*9yeG)nXxiE%JdD@VVk;!}rH;WWlkS^waxV$OE&FE*-zOMKVpS zzq6j=QXB|Y8`-$psvKhsc$+FW_Zl9O$kgpYsf=A!o_FK7vn;Q^NyKIrw-o5=kLM@p z=;-6N7{2k@WlWnS`ng1uh%F99;e(=XWE^TA!0HNWnUmB+crJfB%;$*rQ>J+t69H=x zwiGC{w~0|ZMi)jNrrKC@W=mTdnS4a?^#x$XbZVTg(64oxl6UB%Ro-k%Z8OCPNG0KH z3T%ci(0eN94!RSVPsg-`{>`P?1;H5AuI0DnCIN7RToeC&P+R>3=V@5464$EgK2n&g z`s}47Bkk9Fq@q1=A~WbmsyVpsHK`+yEg$=%Hn5^pq2-o-l= zZxkPABG5YR_T!y-G~Pt`)wjR!G_bMq&Q)jEBj%!I;UdpFG;FJ{13iMO!M{{?-QQ-o zD^EFwte(Qe9?UcZ_4@HHCJ&lgb}YR*!MQKx`pP@E@yKNzc28JBkQ92uGN@N0?TITvI+WPiORaqG%m zC9S>Qz#m_qAI)V%(!Ik@qoo`Xx!3QSwUqip|KZN~hLqB)y8Obseg||%Cwqm+x5fFV zl8e5~T6Pb5oHI7jdK&BME%vI2$j`%2%6Z3Z zeQn0gOIh^1IiRHni!gsWbINw>-R}5X?iDfL=H0Tvq`Au8Rr_U?8~5>y#9RAPb+5N? z=Y|{!9w)RD4p#eYAU6HhERTHiwbO|h3VccRrSO-(hrc`(y83l)byZbL%|UVs6YMaT}cyTluUD*+LfIa@fL66kgW~U^o*` z@=t9}`@|;35phlJtK_N&ot9T$`ULF7BE-k?EivOL*;}ebc>l)gMwc+h zG_rS+oa<(gh&QGZ>AKXWZjvfm=Rtr<*zA8IKqhTv;PB=MWmsh9|uc*a@xe8K7lCG%7+G3^@P%fc{3=7#K zQ#it}{j9#tft`n|^yS@Yf;rbZr?qH$>JIk%2S&@dpP6`)+r|VeGpnz>#m?=2$4Zkr zkw=8^u7qWOOljXKth-s|8+M6^?_0+YP}axEX|zWqP$roNT3fm>P?pUjC8|oSw!?_G8rLXm>(G3vxC?GL%c6KYi0BmaA(0j%BCI>{ zHvp=qYNhu^J5RdV#!?53>$>*1)h9b{PrdkdwrSnROzD?-`b?I2FZWAz(#odJ9A_I2 zFiB8vpfTdGYwx17zvoIeUnl))wtNxOQK%WEse1C1vw*E8?seD<`-3u3CNoEBQ?zhL zi|+JuUP44V2a!3h3TxmOm!J3qu{C2Tv!$4jh$^_4ncqi~k%d@7ZULl_LW)ZdE}PJq zV#aFJTl-6s7;1`k{b59irnfsczf8sNe@N-{AlxNcVPo5~UD0{=VbFVeKRbeHIA9&s zX>z(b|Gv)qjaSyx)mMV*U)c9%igbsi(3#aDt9enZ?9-aed*uL$E*%hu!jBSW4fmXU zLhZapF3NSEPfoUeo553p=N4m)qf_8ZZGkGO2E$`r;Yl|8=0a^r?{|_w+fgLy=gRhb zzNM9F$%nVfX-F_lUYjeVxKbS?wB$d+& zT&(d{>!6Ky?}~D7E~_4xFft^}M`-OSfq z@=|URaucywA~I4B+rwF0W4O>Xm0}{B=CTZnDjsD^`@jS38m&SE>zKiS33ZV~z(~ben905|H27P{(jVYBh03>qN)fA^_Vln{|2H>6 z94tYYl^fd%TXFy;w~wB(Er6QzH^{aGk}W-wOA(&SN?sD~*@cD49icwB|Eh+f@6iN+ zBKDaMZ4l6_#c%T$(Mm?gqsOWb(VJ=mR~R|lb2(5q?oMBjTiVG;r7aSM2~t{#cCAOW z8Q0-#E;#M7Xp;FL(638$xxUH^2!yW2zQ=mF;1q}yUy2?@{`dTjT5{STh4G*)OQV9aSto>P1;#`j)yF)nz{bV5}%!7 zE0)rgQn17cEUvoTA)emM_Lqw`K+~2fPVEHf?wGLj8n4Zv(E-puk?4?A=#O+5c2~(3$(duc+?RN+ z8ad+nJM@)%d_?@6K4uye>hkVhvVTuq(v|=jS;8=>h7O@f%+MVhkFJiJ;$*2dCDcJE z!|#CD8Ks8EYbwUk)uodZfaGA@ru-bSei|!9!^JxBJJXR_2b8`RhZ;tv5YIU z>z(5UM-rc!ET$dg9?UonO3p~-A6~AN-ok|QzymMqP-SRvKU6)|R*%mDhi8~Im75yb zaA!Rk0)x2gzJ!`t1h!7mOGJuX(U-f$gIZ}fzj49@c`mgJHon@*o0a>R1b|Ve)>}WW z>=9j&w+O_qdgx(5^@itq84`vTH z8{nvTeMvffc6N9cT_B&c|59}d(d&>dL7`uA^mXxjPjECc8Y7JAr_bPy@QB0Bhs<`m2Ce_GwlowS z>zvMi3=bfs@gRJavjwLqGfa?ao?;D<+SK4>ZzdhYf9?Q>_&Z;_V`LLl%N-y~^#Cvj z>sg{&cGKmz`aYWwpV=?-N@-Ao+9-kaGHT2D)r>KhYU}hXayEk7vgH5NeILW)>vn-( z+6Da5EdryK4*rbV>C{~p6VHGs);Xs6nT@4RmUvf-ZCKtTAHC#VSn(rvjJDeAdH!(} zM_!r$@OT=y=(nSCp3hFQB(cr7T3)~OR`1~>cJI|KKOOCX2G(A`7OivY&#bOgtZm)w z-?W~oXQ%9DD0|!qyZ)YbpssTwc5iVzqqcs8v&{6nRb#*el-=cvDrBc#-txoKI`W@g zHaM?m=Llc>IG*aZGyaaXfY$qm3;oWvYM$?ohSYl>`8dUIdB3#)9`06S-^dRz13vf* z-g=IP;2*o&JkA}fWAF1H4>2YBu;=YsC4M|N$9$JQ91t+P9iA}_u(+1B4FT&83_E;c z9{KQe{xJ5?WeKY1x8G|??(6!x8k)v@PM4Dt`(CwX`^a7KIXZ$*_vDB(jcEL{Pgr_- zVXaZdZczbo`h9<$yYKm?pEvqFHN6z?n=fI=w@QBfdN`}ZR+z=`=#TC25>3CMxm4#R zoO|LcQTn3qM@Q|ZR14xPC_gUd*Uo0Vy0P(@w>!O$?U(TZFIH7w8*`ExbF1a*D$Xl$ z*ITG^ckSie3z)OdDOY|kP?C1K8};kR#hCuPIthQH?;H51jIxnx)h3)p8| zfB6WyR>*zEU_Atnl9_nBN&Ck_+WtrE8TaUY=P?bHW2v`6FiDhQgBAkdN7zelwCtA` zZ3ynlb6-$gUrvqY#aZTO)3h*fvy<^(L=KI;G0V>)D|?cqK+fY~mL28`dSs~tvQ-{> zWcZXXi5l-zOykz|%m9|}&Sxag`MhBKYQlXp0eG_l%*Cl6`x7AFknXI|kEw&IQ;&Ug z*iiL2q@^7*<2n3pqV|~TZaXyr!bW{4rIVKg zkx;1^rpIC;kWJ3nVotRW@ai~xQkD9ad=r&co&~ixMQX<_v~YJXE;1qw+XYkE1;Xb} z)#@FpX;81JdFcHXjoeNmBKX4(u8h5`XU=Cqd^-ha$#S82dd&R~ngHaQ-->Ne$EOZX zS>)ul)ysf$vk+{?>6rz-w$#tl00zEw8rXAb09hi})G`Ufvw@hEQckE;2Wjc{o%$AI z;n)Hu6Na3jPL!tNNyjEq1n_0Q2apkR*PvB-*xp{E2OzQU@d(`U(I3tGldeI#xd9#$ ztzZYWy|iY3k8Dj3xgrO(#NxSVZEw6{kS;}`|7`R64((J!lElybR}bpaZ=1}tFE?A? z&9D`9Ea({)ahLSsgAEe{`w-IgD!)Zy7T@QC7`H-M_bBqZ`lQ8)N7pe}=Z7ntda$ZZ zd+ch9{bx)2NQTLiZ~2v;dxn%L85`6RvJE2BZ?b*kWH-}pYvTA$ygr17gc1;T@e@B_q`T$PN1v;v*tLch`4+32 z!oNfT^Xx_8mB}3~X5h*-1M=-={M!yiMDA5H`i6D2XCZ1$0Ef+M^5*?YxgqwrNt)#& zpXcE>s)S-h{oD*LVs;xJ;>x-|*#oRv-qAcWBcpo+!;|#0c{Jf=LQfd{dPPH6K91Cm z@otd@H&r_z0$a;c7<&T1sd^TiX)&Yvj@wu>F4DWS`Hug_G62FhJSK4=tF6CWgmcaM0?hdYNCHed z*MEJcfxJL&_)l@qApyhcl1M&^IYgLJ#Tu$>B+^)k@;u0@%p%0ZSX5zZz8~kayuwIj zu5vL(^c0m9GRl|=;Gg17zGm%hFud#GA++Xgh=$IG=L0MUpsXMumy9qLnG?In5Rf~teIlP}eGhj?IPmc(4E3Y*Z`oK-=>*IS=R%vxXz!u^q`QOJH zd>Y4>(`d_Hvkf85%Jj(BZa4)%%%%_wYiG=(LI4RFXdpl>U8hrFSi6P94h?{16Efqn zh{0M00P$lZ1E>*cx^G~$33c7x<3-#m>HJl!-iB4h@zs~e=kaRpz_Xd{V`8tS?=daD zTkGZ_c4I`NnC$i!X5Y?Eew=*2wXqTtimX&!#LX~^s>Dgee&JCxIjDt_u-t)%zXv#7 zQQ{$A#7`QMb8VMTfYjXLiE(rlalEp8?uYKF<%bp1OWsEVZ&?hv#Z0Sz{|ilf(aVk9 z4@wN!3N$MYvopqw@;kJbNw4gEi0VwFqX`8&6=}&*9fy90ob{kbo>ZJSU$lD-%|^-( zsifWS6(rXr8{XOu0FGF)RWW36ZTlbUh@(jW%2!c-H6alLX5K*^G=lfG10+SqB{N^Sjflk8XvQ;Z{|{s{LHF~dSsQ0 zmtbg=uKI7WSZ0?I!w@6xK2%I%JkRim7?u!TJ5PxPh6P{-jALzxc*Zjs_0Z@G_F?@; z5ULs}6)I-V!ld)|XeHNlFkf;8=#$V*Bg~r}n}RW$H*^J+cy52M9g@St#!}e{+7VsC z7dcR&!BbC0sJk^r?H6V(Sse@~7MmEc#uyf1)?r`~ww2T*1i|@`?pUwn;OM}o-C zR9qK$*vyf%UHWhqb{<7pe8F)y#Iu*39=mh^Hq1^+gq9aO0fardcT87`M;l*SGfVKz zm)NFw>1Y}i1RdL7qHrJ%vp5Kz?`nbTH~Jdr*nA5ywx2CVBbH2KhYBju)}h$fdH7@* zTS@6Qs8hpuXxoIl*zI*d%bt-1@m6oDqv7-7V~jzb476{gd#*t&oOD%hW0hU=2FM7j zAkP+Ng0+>(Sef1u^>`;J`F43AUoaDN1t8ZGxYVVsKV@w)>%>vF0!{ksl(rABWwOlghTW3343 zsU)lUw~8adXSD4^x?p)y(Xi>@DWn_t`+OUf#fW1;-)vWH1$%*U-pwf?^6#x&Oc~y8 zFRvWYW4bl#kXS_*&A;JF$dVP?G~cjE5FS%jTzfM|tlS*RXD=mc~c@A$}dd95M&yJ8}3Hg!^5vV5{SC+Z=0I$>GZwX$DoehSKbH@)(z2`RX3YI^a+83wq zyWJmK*fZk`tq5P|r=;W%VWn+AC6$ew@S}~-p_IV6sM-@WDze zpSV4I#+lYjmf8!CWMteCwmTBO6`OV`LTq=2wuL<%Glj?LZ^E z5Ws8`K|kRgKXZ2+eedSnkw8N+Eo{kj4`j&hLEI(0OPbT&q@oYgf`Dcq<3IQVxyK8{ zfQ_wGshh|g`G@(+=^`_~UOjj(N$R-dnI;Xo+p+M5|-=7{5hBUle8##M- z!ebH#QRADj!YjKIfv=Em9zA-QM2xFd6ZI`B_}KZZGRc!(I_T{bIIUV)r(b7V=ZdKt z3`7wzY=YQ<`suvQ4j{~|t9nH^W5HA<8%8&8Yhc90PTYjv(#?e8r;*|-fM)g?fOt5B zNa1FAhk%gkjTX&!tf(XPiTl|{TI<_W&;9RspeX4WoIZ5cdOV3+04l!E9qr+Q1Ig&b z*pxkBJqNlEXwf;}*8bx#2G|2n2>@(SPgk zB0p=E6j1yR`7-+IZsSG%@>n4M13*kcuseWcTMLK?5wUJiKJJ;SGuRzA0pJzSKA!Hz z7(_(R9=^S_BL8?Y@Wm0ab__6OAiUyVi$bSBJ{nD330E$$o59_{*C)Mq&ldp#oaqVE z-vL2W9?KlOuPh<{$ZJ@oIo9e!Q#~h46{>yX1ba0Ez^>ZIeY$JHGT))`7dm@c38VT5 zlF+#EIsRJc(Fk`1x&?%0FMU~Ncxk{=>WNgv{8dfUlg02cLDN|;e7khVa2XT?+yNiz zu7bT**41W6y{v8HQ}y$PPsX3qYZ$Ob*Kcn``21ipZzcfNa9{dF@n_ zWpKx@m`1=Nya9la<03~i{w$DR@XhZ)wggSCk7V-on($lA!;S`XZJC!lpT0O#7+#m-@f223Gl ziK-5?sI;*{KpFm{ETf!0K-l^KU}9SIha(YvcgC+lK+b4k1+1Rc3?qlq0Zi{=Ug8qP z0tdXuy4@~HWvh7{qIuBgRW z?s`c51>y!f>w4+|wcuZ1)O0`vIk<)lpp@{fJQ;lzw7l}jQr>P7IP6@c|3QN1Q;xju%$-qjXG8{f-(q&Z#$$&8dw+iH7%3eeaU7Pb-xazM-)fT|wc+LZMDi_dY zaMObt2o$`$@#S%toP^Le5+fC$(*jk$;Di`4nuW7;9D=F6&0D3NBlp+=9uJ6@0KTY^ z=n52F{=OdYD3|5u0q)h~Wm%Q##FVY{-9L`r<=S3&rR>RgBf^!ceucs1K4lO^z0JY8ojvP=seLhNyxRj050z;}j&l9&$A>GT zg3s@w_IS$0oC5lXdA-Yr&HxpgY032~#r2M(%ebxH!*Q(HK>^Ym(-p;Z8%B?a#@Flf zIvaVN0{H#%5jGuEw(DsXrdOm^5rY+;>UV1WGuU0!*QbY3g%Pj4&@KxbIT6`0I~s%h ziGzXR2~7Y?kb&q7J%F!%Hp&SnSSa1y^W-+@D$)4)4rnQDLPr{;z2m?BL1{*wNSgM& zJvxgDoSKloHiIyBB2mxf1H2mNSe?KZy#LVhf4%H7IMBhk^g&OZ2{vAwEyoa0pyQFUkN%pDN^5rs;~ywiSyZU728|s^))Q8gFzHh{CCUh$+TX zED-IM0szl>}D7gY&wE)qH=@VNqh9O+9OVHt58;_M?tk9mt-2*#GjO zm3R3kVm~7dVPwZt+k}OO1`B^2k}hg)8HV&-y10nV8N*Kp85*-SpDpB^$!2v$Z7-y( zp7J3gYf|0if|kV3?g+l0(R!A5C4l)3N=UVsW^VA-YIKq1R}Yp)={PNzZJ1YxD4Q8o zZdR@->MNhW@lwu}W&z`Ub@5@X^Vr6thZxKNnhQ|?y|OaG>R*g6}W5CyGw1qlRKz$35=|w?4V5@ z|LoXS{rw((p#6PmW|#B(UIVYiDwfR`>TFlTUwbyVc*oCqeoUR~_Msf#px@?6q${Ug zYHn!dVOqjbKfkXYt9s{o0Ag@>UUaoB%nci}XSnh(A`!vYmZ$kB_SN!8!e;xC;af4n z1L@;0R}O&ljt1&~<3P*DcoQ5{Ua#Kn#I^=-hEGD+f$2_RP8MU6zP+{aL0$l^lwm3T zEd|8+&qn&ojzl1Z3laM-h|(47{N!-#IUxS@M8ToR;eS^2N8Avgh1*gNYP^KCjqC0J zA@Pq!L;j*ZK?s2KV+#%gHx$vO0@S44ArfS32Op2R$h%LKK_89tzf9G-P88Y5i#wT| z6gEZ6uO{<9!7wtS+dT5UCz-8yn(M>a^4iL` z!A1#>ChGI_aJ8EvI(}q1ewrze&{d7T5~1Ay)a5S5@4xz2x=XhBx3UY^`(?Nx zZX@>_hqW=BjUNJ*9cbW>*SfIkUHg5~{dz$!7=z6p0-8v#sb{}%g~6*kdMdYLYWSh( z+jZ*Y3iO?Sk48It$lLw8qPP6(oc!niXdA!?fn*IN>HtW9rpB__nQ<85MSx{g59wq~ zbKw5+W`94|&@*_z`6pW!=Pk@d;Ia^9q)cAEsMnwhISQGA)bHNWmCV+lz-SiX)@ z{p~71r;xrJFerV8{;^lRK#%1cJ^E*mXivk;f8-2*4a))^0HmKPKqhSyU`4#3vjX+F zo*kwNO^QnN<-fxMkM@{~!y7Ho*nA(jzQ zbkXGs`G!9XY@DPAAjS}jR)JI&{yQu9z!48T-g5^IY8)u@EyrSyN-}Aa0{2)|t_b5l zlM0~Pkh)U;gacRge+Hj(0em&5Fi*E<3rH02f!bg_NbM>AzoUE##52Ew`iu{tZ*oxU zi$=(Bf_KCfk-Ygk-2651Lae~w8N!m<8VM%2_T%_Uxq)(hllo!tBlaBm&0-z(;qUD>HXgXy->EB?>J4-rF;L)`yZE-j`BlIS@H z`(gvsh_#Orm+imPIaC2x&$~Xlzy4|D-Qi*!WW_E9l)777&;m;j))oBEyb)t1L||;f ze=P;cy&*GzK39D{{Cf}hXTCubQURz)3f|<3qElfNIK7X6RNS052n#)EMve5yvHo+# z5?BZb_&JDEK%W1#>QnAS;xCY8(9I7KLH74c4DQ#;hK>q-dzI0@OdjwsQQ)fy<7`~F z%t8Gi5cnjUU_a!iRrdMwL(E@SQ3CjFkU##+rj^el5iVl`Caeel3F-e%<4tH9+w^Z` z|L4a;)X+3iZ%pvLrj!amx9)eKI_anQ^zK(yD8Y8_NcJ4_W;d zYAE|Z<{d7B2hAy-@Us_xBW8HWMR~LD^UW?GF_YZ;=At77IXhhl$e@P<(ZOxS`xvJG zGhf75BP70%ZXKXpt_t1L%`CYH>|5`GLh=7vqZ{Rq{+p4q2HS-Y2+Zptlg_(0 zS!(^QIJSljOFoLw8w4cfHvr||s5y8getz)!xQ-mKDwWnCk zaWSVQv!1ScdaNA0v+S5z0ZcF3JFDAnwks!+bGwqOek-7yaqvjm8f5oxiCvEj$H0O! z219>{HA6s5uT}2@)CkV@QUSS<>5;~WK0Yq7a%{7TZP&kF3zp%5MFkw#(gJpDt23__NoPVr; z8m{xS{Z#MmxJQsk!t`fm!u1$!2OgzdQ6fA7n?vg)UqbscFe?*}d=wLY5AfzY@JUI5 z#l`5~pA=;`0hu;l@Jn-07|+FYd2~4h95$L>YY8AN-VJiw*=cH}algOLNGy*xS@r;!!I<;hs8AVIXj*iW*GT?- z$Cxk^P8r7T^bxt|e_neDu~|e>NmfS|_011Y6^g+IFbp(av#3SG6#iUW`K`=&h3WEq zbC}M%``)ESuN@Zoi`cN3z$hYu7q0%X%HUdXLqGrXFOt7Mnzz9`b2gu;(vYY3{`S$` z)8H!&mLek`pQdJ>1gEQ3RxtecH^EHEV)tb7^8*}1RA0X2R9VX98q!N7%AsIlSj=vb z|F#~PrW;tWl(*Lm_Wl^u_Z*B6mRJ3hXQ0d#kIx`wGWO~&(Z&BsdKnqUY zE4>I>lpu{rvqj5KtPHK^r5<;>-FaB}?r6Z8Mm#XJ`+~UmTrRbWn7;}YWjTog9V2kx_R?dH!vA8p}I$P9z*BfIz7*#4%>7r#Cn zul0!!;a?#)A&ch51;syB3+H=md;+rJ%OGP}0DIMR3zh<`pEKLdB=XO6&bxp)+019S zANBhqU&F`#2b&%{ODassOUs7w{qJ8H*J=Wl3T7ZpnF7>vxJ5`Ph5rmC_^wRt-Y9v> zb)QY1bZ&zm468U;3&f1cmkAXn|ExM;>0TLSr~6wSQrDkzI~lX2sKZHf208!@~)vL@|Ke!&42gvXx*3k8dbJ~e8)dWIN{;Oz=Y8X_C$Y7 zu!$TpVF3a!tlu9+tAa7!5-%6HRsyW+HkWP*W7tK4l;L8v&OS61$Dg6d1cY$je%96X z@X9ifh)#*8y4E4x%?)c|1e1mzE%et0=cS-Y%O$)2X8@ar~ul2PE1nTVCF+_F*3vDHl|D`={+I?uUDQ>ljz33V zTYM8(w#EO)*muWM`M>{1D5R8CDpV+hGP4yji)8PHJ+o(=ibSC!dqpT)wnJ#j=Ga@q zCfOnUuA7|Gx6kMIczphOzwgd*?)!DW#`U_c=k>gvFSuEr`=Ukn8;@edMcDxC^Hcq= zBW68FAB%YWfO9PVY+@k+Fc;CmWo9iW%{wxqDp822F2q!~?pg{vjMSsw)2U)5sBt}{ zK@8G?#6r$fk|byA>eYg%8pGybIVhjD(vd%%phf5Q4ZqJNFMJ4*WOdXuaJ?jci`Q5&MuT3>AvH_YJlWtMCzW9_wF?Y zO`thoWYqK&S{j2VH3vIHtoFRmFFkg^d2*a=>T_)aiTyd`z>>S zPvX_-9(#vWCA!Qy`JLM{^$;0&&dRhTl)+f_)_+=ZMWA#yFhU4jDO8cGR9e_+g4vgh zCcv8da_~-PxvCj`xkt=nGNskh{xd^-A!OfAU$o!KMuHphpvD_&qLVDJ3{|mxdv*p< zk|~LhUQXxrP2ZJ-v5x^Xfn#r4Crm_KemsL%&s=Yy~e*F_xAnVFvb zHcUMD(CQHzX3!P9+lJa*Jq)^KESH}M{~53$Cs-5)2^Uux6^6j_z?Qcz__4xkE!nIk zX=Ly)QRt;dcaH5ez%ICIy!B-RyIkmLWm>O+ayN&TX}1H%E--nmE%ECvU|tq4Fw-KF z{@q#L%edYlA->5OvH&$IG2TWK5?oZejz|6C#XM>eSf&zD5yrlj+M`{5`;i`KQlcfi z@8*leY=jqNV+w9n?_U8$?0Sit>vVA*_8>J}3;S5IQWOvBr7PZJ3-j?S3qB_Fj!ga< zf5yH~Z09_dtd2kq>3eOMqJUKw69TSu_k24sXxnKBxx&VDQT+BNcMJ6~kS@^X?HZ7h z9&Om$hz%BD2R+xa5M!9>)BSdN(}QhqvMX(O%-@%dKty|~`03R)Myfw9E7^FIc)JpA*=TY}7%wU0x z*>3I$hp1RKoJ*gb*R=6jsRNW_6v@oMT61|67?}UiReEQ8OS<|TV5&I)RS(JqV4{pA zQG~XY>qSbD=Pm+YMYt4%sO$IWV=krJ?9O6z9tL@*pN9TCkLL9nUtNugzDtuf&K zIY)m<&xlyGotY2ryZ-X(it=Qw;sf;WW1@fjTGSaTu*t?$wRVT7wMy&~#~$j=k2Xb2 zb?3(%cmuJKd683Hj9^ap#wpz*tE65j)7oe9j*D%Oi@%KICy08lTKMLhHU-QF8_alS zHBFx%Bo%VV-hi|rb-_Rsuj%i_Z~Bu~otm<`6ix#J+_xp_kAF3^os1VT?kV!RQ)HUZ zD{IOa+cfRAc6cgQY@|H6qw{8y5Mx}^{dW1W$ee3gUu|_q->vBM7SA7Y?NeLul7Dqz zsXQwu`0b~c)zxb#4Ge^@KeX||n9g1%CFO(lo$g*UN zTh6aOKUfwGBsHx>wsNm`RSV5iK?C|Tpc_1vSNSc(Wyd){BL&g z2NwH^EOSW9Uy3nKxqCEWp}|RY=72+l*?tr=IczOFN9g{VeixDH2j?H=X}fne z%0n=W*eL9~VhPt3GMcF`uEDFlzgK(DuLi5>!Z{e)=DkH(-F_E~Z|57&k2E}5rM!_b z!}tLh2)vAPV}(e*JyW;1aZ^Il%0m-MR^C` zh#7ZoeE!{B=jj!r&TfaLYinOUe|S!p52(#~z4j{Z?2;XIT8f;G@j9LvBxZVwa+Tnx z_F&)m3$~z*Zp$Oj#y4zHj`*m=|TZ_<* z&G-p#&BEVni{3?a7ESSrRuWcTYi2)2%W^jK5B*xsiEni;%ARi#FLqkE7D(YsVlbMS zvq;!9d7cyH)$HzU)o6LCz5UyU$I7Ra@TJ0ecMqjW%h$_J9i*Z^rXGsD3|^YKA!ZvW z-YKyZ%TQcR7`!LGb$->MZ8*Zv=H386OZh8N6F}B5@%@eX!AmoPwn64hdYYMMRi683 zzfrxfKuy#K*h0;f{ELP=&IJL{JJxenUFQ(#iWD$WBaF9hSa!)!tOI1Anx(_P8E%g8 zeFu`b4HA-66RFvm$-Vj0U9aD(wI3kU_*KrT6l zKRCa5-^&dSR{9Pih84AzBz^SZ4yuTTp-In_OBI?8M(;TLuCD zgFlz>?XT_2iz8*Q3+3Rqmv5 zos(9_4IlnywCP=%a=Tu$%wqT0y-#SrmvxR)d_JR2?}5z1ATOIHb1x4i#<=RZ(by(y z|9UVQ8KVUkdRsahi(*G(*K<3GI|{28g>|PKdc(6`1Se(}FXuVMS%X#`gSE$mRf44$ zXA#Bd{0D~oPP0lYQs=BJ(@Ex$!mLRvc!l7N()>Lqf12GtmuNFw&@7C~xqsZD<8{;Y zi=aVq7X!;+;%!sbw`a{Dzw<(`JuJz^Yk@^yL}xs+M;D!j&U-wWFUgOeFV7E}F`@fy z8(?6XR8=L11#XO#P6?FC#Ew*$N2kUgY z^|?I}qxhoMn?i&hfma-HdTwd81Zki3YvjmRdSG#V5{t#ff{l&zl+cIxIz3 zeRChsdl_6IFlFfx7vDO%6zr8XePNP>p?I{g5svF<*oYV_~9Kqsj!4Fw6ve7 zGP3P0(f5%=_nP`|;K40sMJ8H8c?x>IpE>r)9EZHX{X|1NFYPgI%io!Hz54fk4Q_0f z>_V|6B-d~BAjjF%+lo)}RAN5H*nH_ol-iTOiXE;vKwB`jN!?<-m08dCdXL(@r~XcV z`o(4LmpEJ8a;CZ1)^J|8P9}i*DaL8?@;AyOR7eOQNknW_QYAj|ux+W-tL?|Hgf+s0 z&g7z_cf@QBCo4OxXANC`_;^j{%w%L}t{Ntg>s?q&jOdxXBz8bUA~(CDBgtB5%`8}G z%_x}IW1pzW%yCyeLreKr{Z$peJ=#|ctwoj|&pW*uEz0(C?8vX4x;0c<)mdav!r0E8 zcu4E?&q9M4(eTL8w?EPnAFEh*3rLCeEG>BE3OC+r9Cg=!wboN)<<%=VI$Jd=Y&#pY z!o~9?#$nw)K7ZMnp>=emL_EIi@|x)+k-;tYd1CN+?GgcGqPt3_e`&`Bqm(O^EZBoy z5WIz)LbW*|hk&r)Rh>S<<-PHHq=*tWk>I$t2XEB@mFY9t?_Ep|qy*h;t5K4tSgw|@LGM9qSAMU^HFaQC=d5L@WHP_U#%pOA^PfF$MX=00Z4zql;7SX!L6gyvRN8n5>5O}I4RTtv>C7Z^7NfP_v5!En|ws|*bI5eKY zSxPK!TyUNZY5qyH!7JV3hA3WHP?0<01|(G_U5s%rU?dIm;L^(jC^ihg9Vxt93g^Nfe#_GpF#8ttb*Abog*B#Ok?p+-@anL3>|dy*{l0IK_AnQ5bEHm z%bx)WV}|is0|zMuT4iEjKibF%eu^UY7+_s4h%3}$-CF>gS9L*Wg_j4F6807_kIK52 zt|nNvSbxzew&4U1e0|wxCW9pE(~5>NM(8^0=J|O1BIQC3h7z(Hg3!e*!R!+b@7n&0 z5Gx4$YEr*l+8OCx1cKz!9!%z=jd7|NM{{0cjB!trOrUhJmZDPk>st@ovh{V59W;Hk zKI-~NW1MA?jt7XrA)eC6FQ*EckP5?mR)bLYD;XMD(#0P6ETPzP3o4dU^3lwoBk^>B9k%YtMVdWhc0A)rq28Y6|xTV{Dc{c}1^~;mz zj?m%e_@A+?0rcpZl`t1BK%=zF_sI@HJlaPclIR}GTre6x>oYl@ZDdhepHu1&~pb;t+EET;C zux2B?IZ9o_+;EVb^wTQ97A-Vlm_w*#LTho*$e&()#}3#*W-Q-dxF;=2YcKP7ku@AP z;RCZy;_eaO}bSbv@!{{;4RTJdO@2hiBx4c<%|PD z{6)sr_E4^ZcWn^UacAikheNwpb6RM2L;FVmc4K;oNPb-!T3ec?B9w0t$bXDSFt!XS zlO8hGEb=1wvCcl^zr@*lw#eo!vJq`|Sc8ko0D&nRh}T8{1Up%70a2a6bT<`(w(}U* zc12udgpxvT1XZ){EsC0_cSeAbKG$zgE>1e&!F&vo3%kOUrjmHcs(WOyCV3P8(5-BY+e zO3S5p211?X%KeMd@8#fVGk;hVjH$J-beJJTO-y&d!6!cpU_Vh-W{ktb58I~A2bSnw zCI@odQbc_w?~+snKpnxOp3X=(a2wie-)$VhvZOkJa~IGx4l9n36wSdN-1u99o&qKH`n1m5nw_22kY^R}BASdrf5enyJ z$v_h4j`P@-KZV;he0+|=FG_FFj_t;oA;~UcNP+NcV;tFZi+uINNp^5;0wTWyj9d`$ z@S>Op$nYJzef0pzRS^2U$A3NmhV1+F%8TWlk9=Z?5g(CIo|ufxFy*>otOau_D^?HhJxmgsR$u;P<2;iIh7|`N?*QBN;JCFJEcJhcF zz0Je5y%8YK`3V_7X72h!UO3ZQ&Q><%^Lf#bUfHOIJT_M|pewiBB_>V`KjDo@AlN=4-c{m`g+|PB3 zIJNVndy1?%SQNvR){oln*;zNcP>~0l6Zng*dy(W#1k}5K_&+JGKXdF64a_kuHoVP) z22^E-NJHYa^|C3hZkfkTFX92$=nP`YWPW_5@iQNfQvRZBmuK7$I#0#bkkV>MG z3s@8+{Pxervlq#ZGEW za`xTj&ByWYvURl#y=UK~*Nnq6cA?IZjXjhmr#%ZM=koPv{G0sJ9I_s4ik3Ji_jP)E z0`0s!1d(#^f+x%`;H~bEF??bWFZNd#=?+a74~9bt>mwR;pV*a!f89w6Sg|Q5#EbFC zVL35^^{mB}3K}%xJHw8t@aAaCUm5CpQb5w*5}_ejIt@{@f9Wi*(LXOpbO>S0gi!vq zm_*s0qmqfq=@h{0g-otJ$vEGR#Tzm}L3+2=kK*V9NwmkPe>{HnZ`-LXnCbdeo3&CdSJf{Do%=7ZH-C$KQ6n)BkDh)g{!i0NAg)7mq@u2FZ-mJGALv$* zpmmC@*nmN>-(|i4A5K7q6iJjb-u;D=RNYs(=ihc!`9ibwyB_A_-;;qR^1_`YPbglz z#8mTwO0F2USm?-=Kq{ppOIMbu=A(mjCw^*fU12rAZ1t=iE|l6K*U>l^B_=j5 zt{@t;YE7ic&dpF-G?6^U0TAb2-Wt@S0*}>6KH>64f8OoCPB;A%<{Q6PpmBBia)Uef zBXA)DdZc>A=7vX2f`kN$+2!iNH~2Zdy*h6pt5cMqj{$EgoQS5#PmKj1MV1|{OQ-{FEKggSI)7s)>@3<-gG^5DS*T;h;vNFD`*d(DxCnA1dPQ6Vo! zoL#aj+~Nd5k_RXOVv3IcE(=a%S)_P@D1yISCpU{6S z-?IxpaNAczh%KZ3g6Bj?s_j*Z=935g-s`{z2qAmieH5=T#0;t>b~k_$_lv}+9^Z_uD^lnBpz%F4So~zjGldF1oJ0R1 z&xZt!M45mwFrt#R?w&g5j9W@u9gPZs;Or9C4)3Q_)e*L)UI`5ztw;DV^2swht@BYN zHff6I+%oUqu`7wXJ74@6TXSyc`RP#~23*hYO}^)&e`0)=mJ>I4+b z4G9v1L}&&$W*8v?{Bys(4`3UnM*IDR5g5=m)1bvyp!a$*qmbUKlS;(3Zxok3Zrwi{i9iC(pZ|5&FSL>3UKr2cEwT`&QYd=h+cEcaX0#r1 zw6&1VR%Uv%jQ7U>yz@oG)xY?K5;re>mWcM*55X>m%&U&)rc>dwm4edg~RTpd^@xp(;H)d-g6xr4Z5X#>;1#IM<#f5Sp z1c-}@qACI)QOQFyxaeocmAMBV1lhfJ_+JprkdN7BdjER62MB-?VUHNz=2UL5BT+UG zWHga5M^o>POj1tD$(=q3p$2;<-;%_&^T8kg^en=+{itn7R+VWkuQ**1cVM)84Et8&<&fm~)*UF^PnVkAvdJ9xoj+_1vCX$#GTpR_2W-JZ9v< zeyl{DnodQD;5^IizbVmzJvs37!)*Ls&1_gjg!b$$;y1@i!{|l)_A9~i5@J*@>gmOf z&w!NKDG2xHl?KOmJOWom$@|Trtjgg4KOT~s>u77y#SPB(>feh5X_S1$_~G==gc z2_wVCn!NR(?f(dp0q;}SWVOhu^?*N8$1Lc4KFMnp$-Ds<${r30tUGl;;_R>|E2Qp& z%otVIOu*$!E3jyl~DuYcR+$ zc=H`dU;%=`2%*EGJzX9)T7=^h=+YR-?brzAWwsbd+qigNs0q9!pCLY3Ut$r55P`X_CJ8M%i! z)F{{8Lh%IY9dB>9B>hVhZnXRb240?2C|U`U*EsIt)vJ3&M*Y;&wy%CcaNC-J7^CnT;_OM`aQ}t z{wV~<++z*kzuX?F1Qse9%EX>K^?Bw(0GoGr-Tequu79ryt!~gyv4zf*tVcMTyDQXJ zwN9{^43g}_L15bpW{42>7%rdpZ1~AHL*@y%*&Gl9VpIrqi`GSg=@s0ej9jMpFsV9{ zYymP%C_?Td2`g7iy&A{fB)*n8KKEzA8cf_xUs{9~*e^|KmjV@x+m$hH6WElI!U%a! z7mA?oEiQhCjPNOpdXLz0U#L;Pokp{DuVZ@-!b^R7f6sjZFL<0HESLrhZI`|~vdzLa zqFS(PbeFm0sQt{s%p&KxzYbhL{1JaQ6oPYr#KmbyD*yyLKSQ!>$IAVg-}hi8k)^&n zkAKG|@-6G^wd$C^sOC0kn|*!bbncD!BvG)#BkKTo>+_XVL8id14})@_NbA?f9_p_< ze*N%h4vwvk5)V4UdzoD`dun3ia=HUhump6M%*|fe4py*Q(M}A2yne(n5$F0?1E9@l zv$3h^dz3qlV_8LCaHx&1vBL3bhTfyFcGMMKVh@OSem{0kvhr~*1zH%ktZ2k`H& zdeP`dUn!4u4wi@4b2v!gZIB(ac&OM!ZpIjB=3wiZ0JCzwIZLW6o0P^v3qa+(i> z@BVfz*PRSdUX9NIf_wQl8;`_6@T-^Ov{RMtWmS8$r)mb#ya%Z*Omp>V}Rrc#K^8fuP3*wk*JlvIbUYp;pkQxYsX#z z&Q(;jKK}^d^Q`r=u<+N4E3sitd_##D(h;G?+HD1)MuczIANl|k@Hn)Sw zP)knARL6b!2d9YhRLG$BratfkK=A(9o#tU|7{}#_aWY z11SP|D5Ov-!+eeKo3d)zWGDxVn3Qq48*vQX))za6K^h_dq2Bzf-a7MyLO3Ty#Ds>u z(=jM$Yd<)Q0^*S>;%}&@tKINr9{9^!-0GaYM6i8O6ZuMQ{|oui0J1QC9uuGaKxIOF zBrEOh<2{U?=LTdj4G}iBg|O~7@ARV;?!^*~!fHSe}Ol}RZa0K7ZnlSkI_IxH2L;)7*yj(svvAb~KA zadjdzT!5m*Ox73G{7O#9In5fgl>&!?-XBZ(l;nwHVOkYhg~=QS z*W{>7zx}nrP=u#;b}nnLQ+`f^3QO8xApIrP(?`wnr!6F>LDY$NhI73l8R_`fNXAdY zH2%&%>`&wseQ~i$uhc$j^3=M_Zb8_m`HJ7v(Uq*i zWRr5f0ag#2dqe8=t@aRYAcNoUQ2ycZsXMZQ8{Du{^abQ+7biO-Dk_%Cc{&KhZ`JmII9@mOzGxW=rh&5ZVMyg=pQyfMc&5PJO-0lw} zQ3RyV);)B*Q=4~jb1=(L#_Qr8_^lP=3LF8nX=*?a9!p@Bk(A;L^*{o9?b1xniIchlHhqWNdgEQhC!CGxEndsq*O;)b z1Elk@uU`fLD&GxJHA0Po)lWjnMhV52Fh=0XwS>7Bn5j%dImfJKZ>4wTYoOgnm zQW?m79KPcw z&dIk(#&I@EqE{{K6f`x3A?ulSlY#$(8l)QL87wDyxyI%;f1e&$`F@vMc(a!>=PeUA zKqxmWPV@80A<+%+I=xxlA;f4H6eB|4G#@#_f%BxdfXKpQi}(e%f)V3m4`OU#(~+C$rlgypS+@}l$S_GF=1v@P5b6@ zaUXU%lS|D}hB2;6mLG5d1N!4()%=Pwp|Ds z*g#9c?=&;epQ)5w?Kpf~A0=A{(9{+G>T)~JYY8_$2CVGn5;lZtY`47K7r^5w9#pWM z2X!AkJ6)mPutCT8bhW3@_mNUPLD@siWiGZW!Q8oX;Gsn}hiA=rwuNOl1{HvyhjJTV zV$u!zvXnt6MBJ(vKv3X|i1UDHkK$ZqXkFH^pw>^jbYD})u}oaQ`%Gd|#!1km>rpEM zU7*`gkzi7oU{`KhaBZ|!r-&Ql9_ly1C%FjwQRI6JXW~FFttnFt1G#6WUTL1T6A;{O z$V80|PA|iftKgXIt!N=BET~*HXqH{CXs+sq?bWEl#d_1Jkk@Ml3B^%bhnllg7Au44 zFSCYFEAlpJ;h@}YC-x8r7|qwJZTRfw&u2c_zh7ja`R1(az5|QHh7Cfz4Czt}F^bO* z^37bOq>kyhFHMaf7}OhMICc3krM~>-z-;twO)YFEy;1YfQ<7^1X0C~ro!PUWenFTO z=`rp`VKAO9QS?AZ{<3BevX7;Jdd7xN5Xn_X8YZs-8BNz!C~{y1#gdIX<3jrD=0L+L zigs&L3H5X2YfF?VKHn<&83LUvM@OE|1TM)-@e3SFzJ;A7?iH|k_kh%-_pC=&5%3W> z*LZ4ziOFdf9tJE_NUUDcKyeHa38WJ|1>toyI9K+l!vrWuozI{%kF2p-8QDicY%mkM z+pCggWIb&WDoAceV`oYs#EYE9BIPPT^mi%O9lJ16A9YJZF~H^IK5jm}mnTFxg)19X zbY6`VdSiORDQcd8!&Uz9{cL0gkfhBQ%^GW;j#Go_?l$WT3VZBKz?HwB|M9AKx z%={2Xuzsp%`e*C_rvP9g+3XtKpMZ-}vWk~5kd8Z?)V9)E15^l^RX;%is8 z=Icw!pz_VV}S zkA@nKXJZE4@O3_3T)a}e+{zTBFIF9E=h?C= zME_dNoeLk87w_LN-KAtW2%H(xqDj3k@|BE*Gi+W>V;<7QV!Gh#mu><&z`nZ9aCj?q(Z0t6zx-6~-IvZbG)XJqU^5UuL%? z03~P;IDFt09}m*CE$#LNV_s0*9z7(6I9e?#4>&Qx0VJUXo5o(fqT@xzZv#jitU^A= zdjpl^ikXyQ0tAjk4`NTyQf!(%m(mP={qYDbSzoxK<_N-X({7M^FIjX}FISSmcA&3< zF9T!NuOhRXpB|If+c+L>03if_yA(0n(U?hJ|1+)r`OhT9Q?HZ7HD;~LOf**jAPnZS z>FXj2aa?533GF4VRi@l`1Uool;AERHlL%@6DGpE-{N^hG8O z(HldWK(dmls_*$y*OET}Zn9Tv2S zB|X-~%jmW6^gGb~DjqMt(v2S1(8b()?Dw$1%wIl^KbFT~L}|RQG+!t?<1MH$aNPIp5J>+YMo<$=6$em$=` z?#m(lS-E1pR?hrOVL%RxF$(Per1<_aJqb4HmTs|4oLfQXOke4BhHL}<{Ts6nh18&$ zuSHTpA&~bl68!KAPAtJ_ti^EYVhWUgW|<|UPEoXWHAmPa@*7p{@_qyOvd%1do>l{BC1tf}u5Mw&U!e!V)F zQLx>SIKAyxMLlF`kqCc;0r-#0IkYrgf?jXDH}B@JhV%NHN-4oAgQ|5=13TqGhl!A` zq>D^z1D(s6UfdIlmcB@$5T#(qIt zdRb3Qh+^WjbCce3tlPDa1UNde_NyuuAIZNF4FrxzVfw2;^@tgBb`hvbw6?e)cmF&a zr!XDHJv2yy$W$kbWAfCE zjsSmD;&a<)L1lUYds=G`V5^yMh?Kd9+7Q3geJ7*{IlTIGIS})W+sw7OQ3TAFd7a%? zKqKjA0(oaBYsJhm%c#aGS>K`Nfm={-$r_T9A)w7GHri*xoicNTg1CEz!n!PCMy$O} zaN`5@bm-lZH_p>K*Y)q;n8>&x$|>0I80sd`2l6Z$Wj5~r_B&QZiG27zj}NyKrF?JV;SaK z0MukRWWI8Qvo*Wk&oY=cM6-bENPvry;aIpKy&qNu?dR*L&QO*T8Sj!UdP`Fd`>kgg zq01&jTYp!0cIL~01U|?*K&dGWH)YBq->xKrSSTS@FpREh1h|!@$7YCDhn}RR8H$q0 z*QY1=dj=exxij`X(Fuq^OD~y6@ zrvoW+qjuj|odbV1(kLdII^|pIa*$ug!g^*f04bv=naQiOiz_CEAqs&uVv*?O$35c; zY66d6vSQj3{TBH?JA+GG0t)VL4P`W~F+fpoSPB)8Ou5KS!7y&2A`?jaq*%;Ip^|P< zEVH-5ytH}g?#BK(g!AWr4Fxg~{u`odio4|>?Th<#hUPs!&%T9x`(FcWUqc8cAMUAeqoa1u<(nLTu(Tj`*KpO{vJyf36%qPhM%#ySpy#ZJ1f{u$w7w^hP zQ97LaOxOQaNwa`dwq;ixa`KhI^@)R-jF_ahvxZ*Bb#s$WFqJJv_+>llzNtzY6vXhL zY5Um$do)h^!m3Y@4#U3D9OV!c$00QC1VXvoXCOfiHz`ZpYJ~PS>cGK)IyB*Cd;3o_`SmgVj zI@1It&)>?8VjX%#r<;K1A+y^ETr&i=Xt+x^HQb&XO`1hzW>ld`cVFYt%a% z73#Sa0_sU_?#HSVT6U((FtL3aB;UulGcMbS3MQnce~9ZCUPRz$i4O_el@2Sj5*Ry$7D91#^T&$4JdrJ5!8wP-wg=*tUw!L5pjm5gRCAhQh4RNbJTPQo4!T07B41pa%(|2{`}3R|mljJ`EmWp}t-^mof`ZLF1nWn{ z&nE~a*PF&jyngM)<+#pxzcRk5WTev}&9$(HaB)qkm}v{9+8fLLJcDndD9y<$<4WEo z?27m?e5!G~(|sT_j2{NnE9z!-84$T=19=DF5Xao0Fhx|M-x5j zcBKB7+#us3o&J^2B;0%#UJyh9UizT?s%8mD0cB{fm;pOC=>|C`=F;QrPcM2?C>X{2 zGRJNoZK(i!zWKq6sc@^M*dq1BOT?{v^w+Cx-k3B^3h2?_#9DU%#PVY%Bfheu7^6Zk z=sLM}iM_`!yND{&%;+WT&n6y%6h31e8t#Ui^i|lR_v%5AYPbY|MgkC41|M9rDfvRu z;z0jH8z}0ed1L9)q{NAk#Ihi~Dx5$^7S)!a*;H`U>KLGbHYYS*T&CDGrwBFwH3=y= z9*5M7%?Ipz4lq#nH!5lJVc{%Ds)=Z$YNs{vex0dbv)|rctrkfoDD_;K`{aC8&#$adKBK zBTmR>pgA0nzx)NM*6?>C&^K1qyzJRVX9K*X$r+MQcmmSaZbYdB$l^JgAUvGqY9M)^ z9y1)C{p>xN{LLca)^;OD<6{(JW9(<>f3ysR*c?sC7#jNq!HY$ay<+20$>Nt->vXFY z8dp9oq$<7?4tG~7D|4PkLt(oH0s|p>uik|xDCW%u3l=8Y>xgO=N=afJ3z@gjaRhK2 zaeKv$Nyh@9)KDveIjQo>eysV#t)ywo+|SuuROxqLeZ#Ci>Bq9i_7--pjcb@A)$WO* z5EAp!N1a3g~Ure~B<~mq(Yq(K9Mb zTVF-(VK~RZJl^GaH2(vUy=s~A7c)c6#TL$Qjx4EPKL#omTC{&hPRx>RyeL3UNe@MAHsW&UlML|)af({T`TMdkYIb#XJit~3yQ5+SJgE}K$sB>-N z`-Vqfw%+UiLvK#$UwF(YF<>>Bl2Od8hA19T9d4!3=JQYf$N@ZK`nv3QY9Man253GN zJ=f?#|5uXs6?9H@A5=JaWeVo|_Q9j=N~<7e+fa*I^wN#XyM(Sg9ry!3eYc#frjPU63^dfbTLrzm*(to{0W$V|Ee#H_j8VXjmYJ83BZY zy(s48rcoe{B!i7geX;rrPDv}pw1Nf{B8?O!t^t}Eh~D$W23F`#Zh3Ex&&Cus&sedA z$qD}MUGIgge};*Pk{Dd$B%?z|wi>zR9apUof+O=}OhD((ojXM6#?d?d2m3YHZ3oIh z;O0pF*ASc1i1W8DcP_R%zXBTj9AuB9BX;VEuJ4Ed051r#M(sp{AmRqBQj(FF6L?^L zYl<*K@vSPhq2w=$+-^dQZ@A4GV1*on@(hvO(>EKU`5QV+SAWBi0fN2L;||4PV9T^Y z<-qvebwoN~%t8+l4C%$ld;f9znP`YXWys{N#)*(;0iXH)VS)b9@)8%uTVYCtAy7RH zMW_NkJ^G-+TxhXVwMo5_g=%DfulpBYx@0K)f@-Bl2OVVNfOKC24nPVN z>+#(Bw9mz{wk7a|Y_I~Pj(^L7Kq(XC?iVXZNdL^+4}MkP943RK!iqdsU?le1zXN;) z8Q}6)_#qfNi1_w$5J0AY8={(`Koh(WZIfP=dL@S0vLh1>CEc3$o^fbiMMiM7$nMLf zns~7?jy3rm**&I3BIlU_PVhiNfm)y!4a!UIf=R}I<~z=wi$WF;2+^{5LNpQAS&)Lz z86S94R$$#5E{XL`yAUIdg0k#d&>%>Gz=#LbkxD_}8Qn$L{s$wink2+{SwcXkbodbz z?wW>f{F4<-_5&-|rjl|6*8mhBofg0+j?+EuK)`e#OXQn5a2(ezN`D5Fq4&DFMQ8RE zCN}rfXHRhv>GohgK;#!AFPh^C$5qD(C4T?Bk7N`PehH8-gM<=sWnPj6d(D{A0K+;& zM^_zg-NKXhcsJhZL#~)IVf5eC%HP_tak~g@4r*22eIvTqb{z{Pv4rwAQPiurDgo(7 zRS@ug2icu)urI;;CP#Y-g;CCFmM^!Lezk^;J9|yB~mQFj%o1~-o zU$rp(>>YTcTByTs-j9Ktk6EP3r=3vz&tFkGLriz$bjn3sR}u&!?}pIRpq3Hd+wVx? zft_1QF6>2=Ez{M~ZW1D#aQ}GJ`9t_$ZeP7a^e&Y``0(_KAubf;SQVfm5J;kyI%5<& zHkguzaBX=e0#c@mP#5?>QdLcf?LL0kwyyT(!(gAgkHzT%`uK8lR6?0sfuw)f6u)wUI!bmEB zqW~{$BtC|-^psZ!!6AM&H(V=Uw*NQ~Yob}hU;lecWDR$nG2e@C#T7yT2>BWJi>-VQ zB|`V{onOEaIsFMyz!Z{)pfzPTD)~(06~BK*8DvkPzFaCy0H1}x>KQ?@7Yiy_P`?EW zyR}@x5$BRgGPx6N{GL}vgv1b8B&f4ZGBtXP@00DnM4dxeVpPlx_|}&w8Ud(_1xK_6 zDKjj!w84+|kP`|pPep|J^yqXv@RtAAf>!6Lt`jhU!e7R_2Y||+GRZQo6evk)2m}BT z8q9cG8Wp^JjIJMk;%$5S3<#y;PBP8TfbJrQ*wWVoLLM{tHc#RIC7CXc_&Q(Uj058< z3{IU2mPALKep>Ob{Yi%@sWzN)2X{wQ!J#fF=>uMpA%gBh5d}YKA1<}yuZpS+J|21x z(2Hr*H|R^pXZ!Gz?@wQTM>K;fj%(trd5P8#$liqm(X19oyoyj#GXGae2{+kqpmEy+ zNFP0oB4u{FLo&|0v#z#>U(y1(rS**-TyIgzQT=5IuPl_Z?finWpAZ1xh~F#oF_Nml z;CSJY`xtKx|1|M7aE=v7=p+6bZbOjsgT3V$>|(Dkq1RppGT_=Df9L{;zQER2{a{f0 zM9uZ`$0VxkkVdE1-aBqE$^EoOR5&Zfo0!qB`79@D; z%GRiaD8X#3dGL(~_f`ZhvgZX1)EU)U$7k_5o2c&7` zH2nC7i!wuBL(cbeAa{ivQ;pR8Wlh|dIeTx{gkF+|NU0w_a{fKO_+xvYV48)g zcgR)-@0Ac7AVtzB_7K!5&J4Wql|(;e4a&w%6^4TdArxFr_(FCh&<6L#cAKOT?$hlz zrAK$xsE_0WqM&0!CjrF`GkD{URPdj2)U?5yJ4y~hpb4TupB^o#nyGZ*+doZ|L&E?w z-??@KzpHR;Svo+u6%)k0pajquJ9c{Kc|#{qeWebHTRWJWS>qV{&KG~o%Xg^lkZFt- z!3Q_eDnm23s*@$(-kSp!GYI`X7b(WoYTT3=0;WJXK(pZ9gpvM-+=x6JPc_Hd5e(+pnCEw4$*xd4Eo>4j?hDTqwriuS^7=yhx@lW1``b>%_TA)F~ zm{9)dsnM%DFDR)*3YE){xdjC4dVZr3z6o3J^WW$nNFf1>$I=wG(%TyrWm|oV_RjlM z9VI0GQ7WrbXbytSM_XmM|GC}CvtUEs4fWxzJ`)ciXr3cE4j8a~f-X3y^X65cGZP{a zuK71|mTUOl{d2{H)Uf|zD8~&wrTeTrV>{czfgtl2v{gS%x2#@sBeNwoaQp%;l zydh?RLP$wfg%hRt62z^MU}6V8lgw}d&zYSU+C@O~8O;)iFnJK|Rc2JIki%uxtC37I zSEHH1w+RJUKpzAy(a_)d_sV2hS(U^9jYc?|2m?Kn zz8f5VD0m3yE&GwS0g~2oKnc3IEpq*DpZ%xE=~qI~;o-B%`@T2RR;2p|8;WN(?c4?Zvmha+$M;q`FSB_Yf9 z9pc%ms!VC%O(HTfQsEH&A*Zo{_@|f#=@G&0MShL#9I#x$rwaA9%FxUSha}Icp~uQ2 z8~%)oQ2>qXIS!z<&~0G%+XJ10M6BuU2XfG@QG)|Z6`0s&Z@2xk7X43vy*d%i1m;%_v6fkCw=vPe>yjTJH*ud;2*%ty%&#K`(5vckgsdPjs0!WhnkhqSZdDADV zy+DCFbFjHm&=e}koQSmBIw%GBIL>Bh5c4kOaIt}+0o4|3 z^Fkz{iE#V#IkPHHDx=TZfpM^B8k&D9ABi`u;m%#=)aQ-=nZeXl?3^vgu5|MAKwS z;gU|1dq@6&<+T2C;oaqBI2VtOKwdrk+b;a{MXpe)luS%%)IWGSal`cNm83c$df}c8 zq;?B|Ox9-!=B;5N5A7%UdhcV*x^o#Exw7sk3mO+Nxwyj9c$JAL�mRwCV;@rxg&0 z3xH^-gJeWU6oMo+T%jPbI%%ci=R5L_k~`nV9M0!_$d^D49T$Kig6kdCq*Sb3Y%gQK z?$&hjB4Qc78v(1w5;?7Q+${-j-2b=)lIgn$7J5!(! zH6LYF0JFYg{9^|kA8Z82hWNE->2|_-eSx)C-`*uEui`B3yNLi#hA?f)D30(DMd>i- z0$y?bNGcpKnMkWo6QMdqh(xCMbQhLKu7W**<{-zeuWP7*Mx+BFFZOolnNj7UKSSepif#*mS;#(?d(Uz z^X)+#-`NAyCA(dy^)i-#54@%%-|oe^@=N^887+d!M9-*^Osqq%RsIzxW$(1dIYv_e z7Ogk{?bA!VOX%)b#2TRs61DnlaD*ZEpwp@C+?VX*U|FC`V}6(CeQ~WmLoW7|W->Vtdg7GWwpZDR2~G z-JX-?v7UTkJ?@aTodRDimr7Z*<*L|Xo9Di|!-aZ7Nbv$fxHcRHaCQ2A_X2hu=tOf- zrD_WD+%ZQ5tV-JQBHQfeQk>l&FI2htriC=@#5nlO#aTr?7HM$_R3}v8eH317z(;qN z^gg3y=IP}5X12dLmuS&;au8RLhzsp`=HVo-*$MXDeOt5_IvTJ?hSDMr=T=r-U|-5+y!r?Rk}V1_48#= ztzius%-5o#?L|&IkEKlMSY{0yHfEQOMBgo~=c$V%lxb57lJBN>9&j@~3Tk=WEcy8y zY`hi=V;|%^rJ2-+z)CC(_guQXZ#oOuna@g@2XrqID2pvVL)eEEQHzAM7nZtiO@p*k zut~`$ddsc{q*PYZg`Gh~ukTf1X;qq^)~d5bYFd{LhtDgnGqAwk&_5Nxku7J_IN=~( zae({H0`}0RfGf!0Sr@iz)LC5hbyg8&8V8m_0XVo4)6{S{%=KOnU|+9*C5lD%_cj`V$dQtE}3IIdX9HiU@zG%cK4 z4fFR=SYZq81V7e`g!zryRXU=OAgPPHYa?Gn*6d}kK1Wf_Ng`QlEHR=u+4211ZpKKh zFgJuJl)6vrg}&Wah=|t{5$Q_UgA3W;Hy1|}Q>&$KF1TE@wnR<_&o;&8I$?bIyJ4`?|i@ zcllKb{%rFYi4?e@WcT?(=U=BtYzULhi%NE9NK)#0=C}9!XA*XQBcKN3%R}UKkXS_K ztpjL&0sPM93?7`?jAFh%)KKBvX>|xVzE3fZ%9e1J=fhV2!RzJFE2i}Mf0S+;lZ4qU zzk^+%i9OYQ8&lJYgb9r4M7NRkj5{9gRhy zhDDui!+>GVRQMwd3mCAvk~FL#O3KrSaJz98$9vb4!#u6mr!)8+pS<~(T}$k*wC7KD zR3Wrhp1LVW|6H_?ui$AnaZ6{>5W0PH;Misq8U4XZ%yE`bS>uT|$4U1S2sI}6e zl?cy^cSM z$PT3lRMr?&0<@-b_ntqj_0I$6xkPT0lV|;kC$2DBGC%!V0ztz|%(hG1O-5|;tP!5l zo3?T7l1nhgS|)=C)YDW)Mf&m-GgSaV8`$6oe|F)8~WUJ*z3bZKb?kJ8z>~_PcnYX0kv@#r0Hm0^&T?8ACEjVOhT|R!@hWeME_oTXkjsqr>Y$J+P{VX3B;M@sL6hj$2 zg}8E6oBHdop#vLDihDHG;Sg+wkyq?b{2G>l;Xm!Ddsx@uJ(V@?LKeu#1Sgve!=?5@ z!Ur7v@6_n|o?#kJ&nt-7z;7k+gt47)$*aCI`C5!)M6Co#CNaHow*7g+2oj<0(NGH& z+lmyYESwUavdt$bk}Q2AoIDbinhRmhg`e;4>4czz)m7!LeYYMKU6s{CZ%R(XEU4Ej z5a&`$S40t0|u2zXR6C6I- zluck?-vq$F^W2G5{IC_4nx{8hL2`QAUmWr9>Ds5}YvVvk7_dl4vj)e-RNI`!GtN*| zbwNc8-hfw2TAi=cmy&YH{mAWID$7JpULNLRh@Vp=*Rn>X24_6O$Sc!F4WAb-8yT*u zbu!eM$oBPWCfsf*X)bdlqEb?NNNR)UsjD-_K_ajVudv#U{k1>$Z-F%u&D=%XB>YX} zML1>|iYeX)2#UZhF(Xtq7|s}CPxHV4?XJ39(rJ1T^hT}i%24EcTnJS$ zsjM~x_4U~B{hhXC*V&a9=rc6~1w(JH@^Or$xFWH*asq)SS$@oVuETq&Td`-A`RLxS{5JL^0pz35v?f*RXWG=?c8T7Phq#UtpQ%;wEM6`2)FZu9<}9YX!08?*1>n7-*RF2$8nj$KJ#U!eoVx7h zl;7==`JECP636fepD_vDNWegx8+p_mdo#K038_+3y<-V-1KCpHs#~PfzD^bItxEbP z*0r2QXCgbuCP~>t5^TPg@mdG-`q3umxJ@C!5$=@aP$W|8x{*zx8HVuw{- zm}u!Tk|b@q>)S0~4I=j$KBH5(a*|2Gp-3ukvRV`+;dX+UHuQ*dga#xvGg9nM(pc8( zZF1I{L!q_WKH5~eoh-iE-y7z}$%D&0?m-k|gG()X>L;bZ;=|y}ow7NkAe7q!l4Oev zfgbhIJ78FX5#okrV3B+|D2l~Lr}{t7tZLzO;{irIbMk6rk%KOp(xlxU zPnQm&{5M{Srw*ntNTBGWKzseUZ%3_WFwO*NKv^_v2bMFuMNr(8kqn>kMd=`* zoP5_xi4L6jc6{PxpBKyPjoB}ls_zo`%~I-@w(*WAV)Nhu{gezYoHEr3;eq`d zbY|;PNqRd(LwjbY-Wzw!8I5PJtr5KrmaR_S&r1FA&pzidy}p~NA608qGHt=em>}FsP6PJD+KEu(1RKyjk@o5tZ06u+eK;(3 z7T+dISU0eE%EmoH?OO22n8Ak&bhF#COxwb1*a19@NQV{hk)eXnIVLG{sX*Ha<6*wD z%C193dycSNfx;0B1^;a<*xgeyBSgbDZ=) zSaze55a*wvUax#mh-Vra&X~HduPiw^lKbo zIX*RMeBH+#GhJULd=n$=ExlWO%`JMAV?Bzcda9HKoCpI4;szCZ&T-u6FS`Y5?5X7$ zVDe_Cp!D8 z3_AvVP5(!&s^)sHlGQ}YYTd#MU^h-=UI)}7VWt}nS1R-^;S>~D2_(n|ARI(KMpd63 zi$SbDSJu+h;JOSg=A%?$R&(-i%hszySmzq`?J7N4i-m*#t$ zjj;_*07ZMcb2(?jf5nuBiX#xNVk##z3wuva_Crk9l=BpN5_-?jbUkgXAu6n^dlPT^5YZ#R(h2@ zKCv^CG}1xr9Agr52b6PqRyBEcl{MO$!cPK?Ax{uPetfD&Ed-mM&~*3;QezfAK!tM+ zECT}n22kAL_T$&TeO1NZX*wr-p#tylmauJ#9Qi61E0POPL6}>I^*Ac-U9{}+Q5Ay} zr^mG#PbgzBy^|n?)#esn*j@}3!s=lp@c!X-Lgu&j%(6FG*sg4-r6ZPZhX_7R3_dPw zd0^D!(y*AmKJWsc2RmU;4*JTtTczKyc&ZeF^&p!WwM8ZgeuOsE5_<^goi1ND%76t? znW;5`?HlxUTDeT>z8|U-Fn@g=*A56;pLGB0kEr!9h#c)3UJM958W1DhGg2n3e?hM( z*vnPJ6KZkdXB}5{@@^ehEk_k8l4gr|j1|sGIJ95THN@CE2EpWb#u@t3pe^IvDhb_- zW5un~OzKEs@2#OD9%VX@>Il-4UHAk|!X5?AV;n?FAU?h9;Sk9sjffLd6`kk}(#-pY z7^>Z^4nKX5iunLyq^P+%VY9F?fOxbDt6+}V3BA|Y^5ta@mP6XgX)xmHnOS~p6VVK< z?x@$TC(90tQGpXFDs9PW5$E%qYpgX3Wmd;itc>d~Q-&uZZ``lQI>Axx)U)|vVp)DG zzVkc$+z+%R>b=P9@#Mi%W!;kr!NI~=Z>~{iU1a61>a4TDq@pBmKDo|Ac&gd|BAdXBcHg8Rnxf9M*+yWBruQ zNhV?Oposx$NVov^C1IwZt-B7FUo*UThb;{Zn54|A9(>hov?#fKJzS`%^zfr&gn}Jq zmyI(sjyJP={gdh)j5s!-qm-rZj)w}=ToxX=)iXx@{z0iq@ z1{xVhK94$r|J!c`M(KTy$tikHCj<0Yy`K&4m(bfGh`!vI2deH0K$}nSyo~D=DoLvM zD=ZZewtjG843f0X&}jV#_4YBbP2sg+J5Y&trUgEXf>P4DUsPcGi{$#1g{=LWctF z3A^Ly_@3g*xcAs(os@{Ky=@NG4bw4f_Yy~RYG*mAt*_4gTYG|H&jqI8#`wFkf= z&rpw+IXc#{c^lWzhN4#*tFp1ZqPfrjT@(~z`!Ela)$rO^7T{V+*?xHU68M8RX&f6q zupY^k*H5Xdc7fp9wYMy5xKW6k#rA>hN73n#Gi1sykz2ZP%r-0|X z6!!Y_;6vk;13UVSPObZgOa`CBw&F?75S5)!cXGPnrO|gq>f!N9*OqgHdo7QkFSU$h z3i(dx#HUo3tOVv+zchC4Adf<0mzwoei&`VL&?|Q9^q`$VL`c{*MFlnYZ^=U+U?4(D z&e6E}u0&thu|h-3i^T6puR<6nU&v8!{}89dV(Kd(_Z~c~p34c}deTE0voAb9_=T

g^MhyNf$c7tK$u&va`g|0^l< zT8^ZIyO|G-LszJv|8k-HKGVE+C?g?zWmmb^yTU{6FyD%Dpkf+d2rnCS0HCOopsfco zJLAjo4YKEpVv~~sqg_bnakc%NR6j+D3PD|py;FF~zncxn0z0KHy}3t}`*vu)fwEfa z*Y~8h!_;fcD-gdV;K`GlvBTF@%F%XlA!FdR@J(*44qL|JXm-8ExG-)QKNcAJ*64Zj zm#sSXYV#g;oLl1xlVmI?h2BY)Wdlx;%gZ9fQ>`mTmk)TI%_tkGnbz%hlpD`}Kkv0Z z@70)F)LW3E=2NP}+Awj1uPf!=8JctL4+?=#@IMXfseyki0Qq#*4_1j!sYX%{v8Is% zuW@+*wLw2IG!g%VW1X6#2QzhHKWn5NIn3g#oc_A$wA%OQRwOiEqObYH^|}vu4FzcXz$%i=k1^Jym_PNBe~K zO2qdh+^B68?VdmBQ+82!$(F_E0@~@@%?}qFbi%8x5%c?-Tz;C_PBJrvx+DfAL88G4!F& zsvjNs!U%ItJyNDyXO2}{U+*p341sjkPO(}+GXy~wabwHV$#J4rp8NiqWm$L}bpv0T z{@m=997)U3N9%K3p&)<4j?*;*l->nxZDB)y5&Bs`p(B{u5D5K)i@ILk!J}v7PNcd= zF`W_%eY$VA?o}*jGq6-~oU)YBUz<7t z@&25(4asT*axp}n#jic%Q_a0IhG6))R?SsjAMIXmeTT}^m#UTpl20Cx{|3#Urs@?W zWZ^sZk-lj$^*{L-U{#EWe8{90eANUTvw4zj`mep-R~l_VoV&oX@k@0R57;zjsY`cD zEE#bNV4D-%l6F`gFP>P`)Jw2R&uvpVN?Pt&Tr<$#lBh{(Kv5QYHjs|uQVsJnN&xPH zaaJms%dlg!tn||RNFEpc>ZXDRPcY<*wZqiQ zzvdf12SyuJGyrH1F77w_&3)56#^5?Uq!K}33p&6Tim9AoaL#ipzYV9y` z=0yf=akwG3o4r1pJ0TXUuPhQ6yxDh{IF>HFFuYn=xtW}l%@71)Aq(LRDo$wvBNm~? z&IU6_pRg4c*@%~pz-x6CMxgo!{S-Y|>*9I0q1f&uTwp{M;aR!J6L)Km46r=*$$;+G z;%rZ&k4b1x>G<7~@%V!ufuU?nQ-+o>sdQqtREpE-r$`WSK7J zyMb3bGWl66PP}kda&_CtyuRnN{@0?59-3KJ&^8E0Nk<8kwp=SKomg5vD1qw7966^> zpCa6>{l0AE;;2o?Vp08)aoa?8gzq5h1EtzBn3OF_6Hfonh2+z@UtV5&tT5E*m!1OL zT29G}#LO~ioK)_RLZN0Mvx+E+$pK?cL#0p>nUYW^3uY+O-6;`@{V*WUVlrkYfy2xK z#}HK*ZxVUjC8Kc4Np8b!fjyTpZimjazKUvI1mk^63@H+ITYnup5TQ-#EIr!}H zE8yKhxO5)uMHcNhoci4lsSuxFw)J1vvor^wT>f0sE&-9fdpW*76rG zgmw#?%i`o4fM4pt8?h$L$oY)2SUTcz3Bp6<%#0V_Teyxb*?t1hQ*6xSYMD`K>f}ks z7Xg8B*KZ2Y2^c>KAJ>}doJLyjSQ|o7&prX&uqPD0;pLF=d95DX)w{#Tlglx(s*4N8 zEQ9YahX*dFl?}4SCr@~!6eS0kX7XAy0 z24@|%oZaCkmY#Hgq#`(6X_7#8VdanK7{_vx)i`ywHfhHCip8eE{EhU91Rdi?3ujF$ zX#$V7T_DxzPJ%sA^mK!xtVI!Rv8f@5q;XN+f60kGl$$e#+N}Z>3&(D#4<3{40}z6aFw%P zY@9ON$>s~p2*2LyJ2i}%ZNKI%mSV|;GQ`gobH5LN zr+qSbNiH;$oSuRJDOmah?-ZMe6@)6V+rG}Jv}U6C5w7akYC7O#4K)mVmrHX}!MoMy&=wh89+ zJM|FkSOKZFGclyT=lY2Y+q}Ly#FT-#YqNM-k2K8g_>bt)D<6uoJ2e~St@Q`=^&eUS z=FP$W3j?Aw5txrZYG%j6_Rzv-`2vYg0u$)=5#DsbAah#ODEHAb8%b(BxH^R83~DLI zIr*IY#zoytmOWhk&{Z_098xOyKbQf+x~7;+X-2$As~fEczJJSkU!_X#MBVYP1xZ8u zV3N&-Ggr>!T(QvUjbV7mYLqBwigk%kpo`bhHK^RYKWXE0sz}wX8^lIMLLBS&5F0j+ z6FKJylVJKul1bw|VKK!_@>eXq7UX*rfU&dKSv6i8U=bfKE zxsC{#?mk!Fg%G9y>dymDX34l(;nUUMR*dTz?`SCQZ%8+k#<8oy4HyU!)%t5cB=gJ5 zH)+2az|*D>s(SJr-rL4M$zG#ZR&|+MJQ_(khPf3$slm7DntEJc|FQxIhd^gGBMyt(X8!f;zoDu<&O1%CIA;qJ}w^pKIkXd(PNe>wk4tB5j$-H}%S8 z(>q@i+F56t(C?z zX2#p|pHZJ6JQk1eQ-pEtjs!##%`gM;XqNyXyzkZV5?DHyqbAAJo&2#IviM^oEAZ%3 z-aWIv1zS$sIj^VM@Yq|e-Hk(YiQnrJ)@OZF7Y>zvOUR!o}_ym$OEc!yqM%P zX4Z{+`E*hS#oJo@&UeQ-^sF6vP`NIsuOQFhe(1)wFYxn~lKM2Y#($U}uMU(xc$Pm> zPFnu;0qrSL!;L|)lBTFvN&ZRBgWZeOS03Q)G@uQepT*vetl;{qen%sfly5Pg8ckco zYsvsRm=`cN3G|M4_YvFni_G9Hf4~mtX`O(coP(&$S3g{pf00p;Ke@h>mv`8UJ&nc^ zY!Ra;C~0@(#XUNf5@O65N@J&MHZvfcLZsLRydpj#8a?>;rN;Jd1QJz&;j7!UFFnNv zx4ZNJCyhkJi)kp<6S0QTr_Td^NJpa* zkFj;;VOZtXutyLUf6O<~~%njGasB;^0CsMw&5MJ^Lpi^uZz&@10@ z!lCVi&z+01^mruGR6hz<>v_` zF}5S$lH$_9+5{H;C&M}tZ3+Cie_wgwuWq}+<^TKXXntVu<8ru%;G=E%*Lq$>^Jk}O z`2Z_Q41NAMv|Q<6MIm{NN5kYg{rJV!ci7`!V7X)&A9Kzr1P1|RfkND52UQ@`sN>=& zvT{^ZdZB90L-dkCcx{)b0e?~L-{N7VtC5Fj3LH%*2H-a)0SC&*JN|jGqpHg*U>u#E zc@TBu%YA16o`t)&*yTl`3jB`l6c#N9bPsut8s|Xi*#Sy~bR<4ZN*C^;k2Z&9gBu_W zqcBnuzK+hIDR2H;cs_ivlF$3z-b#C_xr)6|Gvzyvb#{nWbKwky?~f#H@`0)6>H4$a z;c8p%P2XzAem6}6E`N3?g|!jphHP6sk(B_h`*HPY$bBUX+X0!k6TBp0s+@d+(dfkQ z<@w`A%b$nk+(gOxV=N%WI5XZ_o9DtQH$%C1SN=r(5WiWteGq;J;7t|M2o+wxT0x%k+w$Jn(sKVOKuH zo9z7Y{e#R?a!(jLj*!rEk@7XafkF2PLyy)?f!lq3jN3{FuXiPEYJ^r={2`?V#g z?^$0r5xKWT^!}B900%xX3h4tcz#g~@=&av1JPtl|9%&Mxi}+5@TZ5pBF;DHQI6bEU zgBuuOCQdA?Ag|ByTant3Zd@cz`stI!K{?>mLsQ z%MSeWOFVP5?T__Ext0<)0er9|>6I1TS^9A6FuPDE7b3B_*6Fz+9&3w(Ai$O2X?SKM zPRi|e>FH*D^TY;?KQ{4ldSzhGwU>?HCYxy<>bv6JtS-H%Z5v?(4&FAPvLn66`S5WV zJ3U5t-8{trzUa8Bi8}G;xfOsvYA5i(WyXLI{!~gcBk1F7cNZ)|_W{v&s)PY(gfmV> z{OgG5MXsFgjenv+fwvMdKeqn567^f&_VZc!oI)-uXkwcqZKr>$91BFVJXk}Wz)6Z} zKEh?Rive6ik}-^llswm4o7yfw!&;b!0Cp!?vB$4NEiP=o0FFmC*w%!E@{n!Z#{?S_ zw@V|PzB7>8myYG`|S?%~eTkmIb_5S%7pdjhak%%cki80sfOMrRJdAjWm^Atu2c{ zx66UOKytS+d4S0Fqgf$R%3&I015+$M-ee*VLk2D9?#DWfHG(xtC10csd*oj*aXHqB z=2fyA!m=Kfc9>kLKROtdV_*oH{`;PTGE030D3y`cOHA#!hBXNLYk?X5ii zW`i5@`XiKbcbAhH0%i}XxuCkhMwphk&OT2&-gqYexy@OGZ|rr^71^F(>F*sgF?%R! z0~W7&uo#rO{O!qMPR%>XDT8TTRy4z>7AR%2 z_!aaXs@;2ntV$HnsL!fLNbKWHrW;4${(2kvW#CHD6VZoh$qqg;Teu(tBM*Yf48eME zO5)$wE%Dye6wECxOLWdK#1XR`!Cg?#vt9W9S*yX%_qr5ZnT;QH@d(P}HXUi<0@ZT{ z2*Aq@*U^lNz)Wi1l=UWX%gFAD6YGnCwh%Kp2W0gyUl(N0qwXa;gG;3$)M5(r&yCt< zWCbTmF}POXAOF~03{vG5zwTUyF}S!C>`d7YcZvSnx7ONi-!hiph2k(7Iv2;R0RbN& zLS-vctsdvAkb@1ad!e`72ecYyaA|YfL`8oc-2d%CJ`t?2xDU=A{i)SKHmzJV|90gH zzKsWT1RNy_e1%vAvw<-kW3MDKgL>S$yC~PR>JYg+9{4%Uo@>D#8=_po+flXiU1 z=3PcN7@B(eRkeI*?QqIO#tqB!W$U)4?vsjziLOXP659ZNwO7D)5m&gHqi|BqBFg@* zEfl$H;h`QG>YsoD{9JZvC=Q0F%(&R*gON6ufbkTcp8ft<;WpFp$&M%&tBVLm?B?P7Yo*L#C|ufCMeMV?~$JaG1JKcOm+QSdz1y8ZoTF+Np}yVF+5 zY5LTcGXPk;kzS^IrNAQFG81X==~^RjV{8$N?GsmFqMna<>=2lLgeH*>*mYVZE_!Pq ztyu=GHTzlZg`>9z5tCV0X=l8@hCIC-RYE(7UYROJfnDe{Ie+GosmQ_;b)T)=#t9^d zMP4=rH@vuR$9h>$V|odMsUR{4FOnCTD~0*!W^SjT>%&cawwt10)2`X9kPaw|^V@xX z=A?IL`I5KEd@o0Dux90Q1kb2QlQtcq2OnX|O76{uV5v^a{gFrPFR6$auY-sbdAQTz zrgFd;1Y}#s!yMQIv6?VlanHQ63cl=flrVt7!+3+zL5(CLSfPL?+70Y>^&pLbA7v} zS}zxlG#iml9_k>@s_^wdCS~|X()6HByv59vSf7ze z!NgjF8cb5Q7>U(@N~r!?y;Nu0okljdR~a>CV`bw}K;aaw7)1 zoG+dFnP$~s@FrJUR;q;U(ls`S(mq&;svbIG=*Cb>rGsD(HYNQd=BhrncPsJZt^l{zqA$9k>(7 zCft={)hE8n?|mqez5YmCY#8hw^`Vg<#C(h}H#U#+Kfx0aO1(3S{zaZv}WP#RjR(rhB}Xl)P&Rcbpl_=4}k!$^0@wWaMT7se6qi!?Yy-Rk05!< zVA8<_y`;GbOR7bOU;9!Y=u2D!659d4_ur<0>GX9&3zmZCr6OgVst4hEu*Bwmhfn_F zpOJ0;)@GJDixyekGtZ_NELmSV<+$zjbg1wxGDWx5_LmMDG`ipkbfl6iP%+F;Gfubq zfq_(|bC-up88EG9qOc#7>3!HFWwXC=L@>;;#)c_BdHj*pF3biUkC&3lg*j9`;wL-$ z$^v<62R0I)Y8q^)3BvElj|`xV*ngagNzb8Y)!7(#{C7_vKgr0VI8qoV`6OsA~jgi@X90e@o;srY1JZ^HN&Z~E)nymwWE zxIxjg76x;j>SHeBT--eXdLcbqTzw4*DAezeL+-?TsQf)^=z)Ahv2WKYspob^4jrJ)*)t&4Mhx{)npZz*F_|EL_~BFa0e%t{mvBI`gjvU1 z^Pswq`e2mfqxxNBHQ6G)XGkVisk^>RgW}z;V)>+bshFigodLbCXv^LByLsv&k*Qa{&-9Je`OrP);+mk$eJZ{mV)2U}xR#Pgufi*s9 zBAY68#fV^v!wm>~teaGR4s1t}h=`#yL>4HL$h>WndT;J070G+1>ShBn z1*=zVg>uS|B}6kX1TZ3z4!*d0szhAxQ~wAtrm=C#JAQ8!M|{<@1|*9fr<@18i^T`{ zi*8U8m1q^k?nNEu?i*KUENu{t@Z#WDX>36iV?HR)|9gag`N_u-nA+YA455CBkNt;` zh*qzVgZ(AHUW5E1rHM5tK~V3?Zl!kM2N#|wk=Jp9ftPz5U;0S+V?&u;iZN`&xI(Kk zG4m1-;(r_8@R8M=WL;Z*Q=rIiN#m`F5NGU@sAAtaB!I2VfU}Y{>dC}%LdC3Xi_0nX z!MYZ$vdyk{iQKB{J;jhVDBiySd|vS-Ap$HyeB;fSc%bO0&4Lrr!TGCbMlVxSZ}27r|$feo!r?j+Wn zKFRnjIVY4NV4K%*`67<0Z6>gwtnQ?Hn(?l7!+T(gmY!}nx$D5*-dJtpZ>e-) zpM-BgWg<6HjL}r2S845#us}H+h|DnMG=mwIIrxdFjc3d_GU{td36ZJ;@|lDS%AuR= zm&PpgK}%!uZ>-9o?p#6x(Mp9gu}KnjlV4Lfy8FzkuKq=o*ZMh-1jT&DcE6~xMQbC4 zU{92-eDHYq_|u4S>z6X2M&KNJmpFK8dZIDTZz9tJoj}d)?|aDx%}+KH=PkVRcVMHd z9w(q;FlUuG@3lC^oH}eKHxXH-`~6v|-W8oKXSL_zZrm017}OEAJkFt}-%h%PYlE$F zpIe0Q)G6mgAln>wbtXL?F_`bxQ7*EHP~pI8s;jr}5X&TpvsWX4Nu^+Hk34EWkSOb1 z9v9NYJ3l}wd?;*~t$eiw@V5PYAd z>0;9TGbfK;*~fT#1ko?gf{$WqYulGTC3inMI@uk{r%&Py4Db;eB6M&~ZX?~r`7A>< zb8jfPqtd7h{@DFF=E_A*K4;K^+92f0*rMK_iKyR9HwoATh<%@@kpqETZ*Sxg57(PnBn_x7#VA$}ZBHNVgLek@N!@0qi3uJQ_b` z1M|gMk1vBY&aym%LKAzhXsj<(pZIsszf@pyP_PEZt^E(KQl zG0KV_Y&nuQr@nr@xaKHhsj`RFDZY2VxeRsMqEAcbv{U8y51s1S@9F+xRUcxBOY zUzlS5ccSFMoEuiz0NzMrYuV?lleG=^ZBTc2gU;&To`t!EBHxnnl6e!;e+7@+U8UY| zLf~^i!v<9=ZjVh8WE6t)yntk%P$&Hi^CW~?iy2ZrX}NFDBg#=s>XQhD)>HSzo(gO< z=N@MG^d17C4&!r$r|myYjXyh>A?ABxZ6U5~6N>web)YG6lsJaj%R2{(>BZRm8z<5S zU3(1P!o|qMDEFOH%G% zBuZ)s8Q***p+GEF?s$u2pq82T>_qi9BAb~uYeNqSHYNQpJ$7Wv3dz4-ATXo@Fc0&i zHu>3{p5JeWXx#t+GR~q`Mza?H6Zqx)w{b!nf2gQZ#ZIcFL>-|4GB-~cS(K^#8KY274AsMZQ0YC5a=2xA|UV_8w}Zii&pw1CH#EdV-V@f;!bPE`VY8Ua_ zohFwM`ww+EeLS?!$e{)AE8%kjyv~;K|1t;A1z9noMl;xF2Bq7T3^#j z5F};dq=X62QpQ69r$by=B%g1971vhqpMBw@cSpe$Q11zk9Gv&pGvV zcUXzhK-P`e=bjtz@70AunUyDdcs%l1KPK@Zf)Su!uI^D?uR6Pl6#5+1{`b5@UK|75M0S}Jn&EbRothe z?MTVKvQxy5okwqOU%@2=#Q9EQ2*;1B(hH$i3IS~rD)?qZm2iSyKWtnv%+I&(>dVE; zU9i}23DbyQf{kJg?|xJPV#S!SL!SqFai<5q?}`6h0kraA-W@jC&88l+r1d1`V?|!i zz(|0ak&MJkIz#obG#g`_6+&&kBW2Ze!1~k6LXmy!3}HFvrkjm|eM~e~^lPX6k2LZNoh16P%T(>PkmQqrt(yaRNhY#L^h z-YR48OaxXAop}X8n1N-#cKpGM;lo+ zbZ8gJyY|f)dM;{8e4YB0eTG3|FK&;@U|FuiJykT*tM>P`WL0}&jSG52Iew=Nq6sHb z>nCIPk|wpiKfp1?R+M0z5*29!c%U`RC&Wo1Ow~it7{&t|H(>1Iq38SLFbFOlNwrdB z-sBs&3&G>*ViNDdL|;!d$!hj$P{zvE*W~0?x)g>*o*L2%qu=71xmeHcV0qMkWq-GVx|WZds~i{^G@~uj+)ohU0TN#mrh&A#V5`Jse|xxa z3xa#)K*cWYZgeS`S(F;TKeZv4^Jb0*(qEHvSy}g&HLz}syFdovVD;wmA#wsm5A*#i zV9t|17rv!l0Gv+}?db{%$Jcgi(RGV+hLNB-Z$!q&ShNOuPamO9BBM%gxr)$ptOZ3y zq8^`FQ?=VW0379MXmw3Zw_)N)YXK`No}4F5FZ_8NWUndCYvRIT?DO=in7-@?!<8mU zQW8#zpVSQrK8~yl3FU{4O-m;XWtf5o;$jaF6q-C(Dr~Tc^DiSaqfK*5J(dB8Kd*PQ z0udClvW8h9-Owli!huCS&Lio0sZ0f_Nmjzxu?4tx!A2pIO?LO6+9374%4vB%=~dIlUaudYbanu0t77CtqX+OGq(}J=ReJ2(e4qx$X@4egH^HUbBVR-|;lggh zV0n*(-Rfa#Fe)xGe1s6rDRU#Mo~{Lw@Jy$$dp9~D2t8;4bv|LHD8?eI#5>9- zMfX`Mj8LbZQ}m#>s9O6xkWv2VhPNi+P@u&r?S$AH7^&_JPjMC4eG4jP`H)|JLeD{^ z9Z54W6&5iMYFcgfz)Qt$*mk5G%nC}J{7A*>^y1VrPXue{p^NA|xtSY3Zcn`$?{E!$ z^}e)i^Q!ilSaaTvOyMJAmpPcpJ50mP>oPM=o{N=soo2%ugD{#a(ACi@QAP zXFRm5#0k+&C0g=fVrxkyu*a`?3Rs>U_!bHs-Sp5hxlU&|V&M72S9s4k#ynxN=5aUs z`=~&HMtc*S;YsHl6nhLi+(&#{@X_k@POgt;r;O?geP^&&YRWOQI^{8?%V;N6xj?pu zm2ya9E(Q1{e$NE9oz|!QXb?d8%@91rS965+Y*B~oYWPAI#C8MkMX;byM`9n!hhHFo zfrWTfvA;c-p0B3QYvJ=|Q(cwnBniEjGJ2G2rE&si6-2(2E`1Z=kU~O1FjZ zKx=ylL!gvn#aV&Ev{Qvrq%y(LMms_ppF5f~Jyb&%-}k~SjbY^P7eZR5A7w=m)m zO=~oin05;B+^BCl5P4%Rx1d2t5MLxi8UB%oME41TXcrJ5+%H5&zd$k`Lnqy!$VG?}*-()!kAsM2-XP=$ z4ebjSZO+s;Vv+;svuUyKakkUs2g(5XqmA`{fcrvr(1k2|pA$PknlUS{{Hxw1BJ|Cp zhi(3Nhdk*~wfsSe9~gqc!^awd`OD}*W0Q?Q+zZz>>1Ew6N8DdxfA|rx?;(B0;pl=@ z{4g6dpdI+(%CTu{a+%6YOu!T98sf3VL&v1>QmpJXm4OhPFmgu|RRj#Jr^ViZMH3u$NFl~|_>`X=QrzGzv&j(x=QYF9Hy zvj^s&U(!1|txxEqAY6f@tqj;Z$$~_zm+Jo?Mu5Mh#(CO*Y&|!F`2dwJKHcu`od@2weoRz2Uec z`Q$X6PXrskfWZFmNtdtDTPV@_>|o`u^0E%;wfz!IJ3N1MmRHU~xx3qU@OSzE9Yd># z9fXHRm^5)TuUnS@cnJb{r$Dg&uGa_u{uCa6@!#+OBi=y1U!2rB1x)YMsTW+oV1r-Z zUEY=hcKN)Xiy17x`c?jmJ}@|;mL0%`c*g?<{SLhC1}5~O(isE0Z{g3y)#Tr+{Em>O z&3c=ljrwI8Y30YC(Ck7=0b3X12+c6@-o^}2-WnFzU2HTz+q7Q12B{a{r-2mt!)&H8 z_s97Ezi+Oo&(Ln<_;3{l--^@M-o0uvllt|If8rjw#&BCYJ^c!i{qILB7ke88k>Rk( z&lzeF8yA3RU<2)1E(8H|^t^zH)_Y)OXuom^R7FOXKYj*nD8Jr@xDszUiyiHHGy3xv z{rd0!^UgPb7y2?#Nyb1xnYQ+%|HdZ58ST3VauH?4Cur)pl)l>i`bKm<-&ifdx5xv~ z10W#ee3RK~_WRcTydF&=u;|$OZOxi!7Z}dSyjSLf%W-(3;=?9!9>|QM7^cIkTnKd; znQ1(hXfxn| z(*YD3)9;}&Fkn0m@#(8FpszLGuk=W8;i&k@U)K^#M9o|SopMKI5g6oCnT=?m{m;K; z1p^wD+E+9iHlI%3*fa$Qw$F#6|7EX;QokJZG}6Fod1MSKmxLX>62E>sozE+_A+X3} z;UDa=z2g=b#`$|B@Z-13ze7}?j&o5oE7;fw6bM)@iC?bELNNJ=`Zs@M^N|lC&uI^c z6v6B{tws3h$X{G5`B-f@8L@oNY59xtFVK(@_z~%6fRM86Kf5FBkAXEk8VQ$Zg9-97 zv}EbvAB{Sqknq=W>n;BdxzE{aQ8a!->~7Tf%V`b7Gz&2=XuZLH_xCf{&M)Z$8%@V! zrM*^94xJOuRULHw>-FFM7J2_4gbmUDl;#5lM6>hcoLmR^sl;81l5_myH){F-W$bO0 z+)~oR-LggU7H@O@TGrc>4B*nQw8doqv4Y~tVbYim7(1eMGK8)_Wm1iqmdK!~!T`dU z()5Ba{?2exhBguY*9eu-ToCr*D!dbI$ipw?11f?Jgz5{^z^6cCd-frr-ko-ZX#556 z1J45r>-!?7VeMFP^YgT~;J;5TX#jajj?{P1ek#R4o?aQ!R&vY&W%K@e?!1S=qOc48 z!v^r_yoybGh_Cn@AO8JI@dj{Du}|!2Z;8C6*dN)@CO@ckI!GZ7rsa~;zkH&d%LY607X>&>7U;;tQ@Jc z3iB#%Z8iiSWF{MDH5w1~zkSW;74X@T?Z#2!w4FKWpbwCR^kD#!4p305)26$$*M%QN zk17Owp>?ied7?{Y_8aN{x})X@>s8l=>yK5c#sE1eFD#;OR)NXW$8YT*bi>LgZw!iH5_~?$OF+& zsFR^BIP_`6u`mY;m`0$?UIQgFHpoD9>;L-3ddN2}k*=dLS9t>&V4@Yl?YZ!It_63_ z04j^=BVr1)3Z})`2cA_IVC)yWY>U%p<-XkO^ei;u`JdY#9m7$u^CF7x+dwEI~>JuW~PCa>&;z{_t9?Pm27!n_5nk^CUmWp0i}ohkUF3K2L0G0RK(^sN?wkWS7Tc0jIPKAtYqRfy+73(s-*0}w zbEDU%4V>@EG8;7hs)fAe{|_@ZbIf;uoI9ZFwLxfi2&c^8(W{HJohnydeqG8P3PxMN zd-K6*xf7-Z_%7kce^gXIcZyCpc~hJ(Y5cMIBr)uaaiuO5P1%B{UpnHdfoz7X-|*oy zTPJx*CPAZu1_WM#sr^J6T%azvgBQcRBV{p-nO0B#tRcn_8kh4Yh&=wikMSz_AdZ#* zdJeilp|#is#0uuX_&+S*ZU2F(3Wxi_cy8wp6$$5Ju8yQ-KSExyJfpqszvERG$I3RGa&IB=HD=5n;#b(1|z+~N`98#V_N z=T2zN&4B2T2YyoxVB8lrC@xA1Wcj)6laW2ks{0yE7c3vk;G-&B_y%DQ{-7LzR{@+} zhphqEL@8y_)=ca!R4LS%dt_a6x}oTU2Y>Dd5g80dB?xsN;r#KR3=aD%-vX$C$X`6! zz3t%tP#GX}IwT!2^qamFnPh{E=8|!Tx`323#ZgVzx$xoGO1Ji9RYEHk9Wto57_SEk%~mi5K@fI;@9+k~OWs~?Gd9H!vr;ch zl$R1s;6Ox*L`g}?VIczYE@lZRgb>+a0wEBx^PROs5VrZZ=X~E^-VvHO1y*xw0X_!n(*JlzE@+d(xQ2v1U z-{te9UyTht=-U`j8A9)Qpuj`m?@q!;om-HgKnM+Pe=KTZ~$W{NlA)oy)PfQ?6g+QT%Wn_%Y(V z!4e=L4aDtb#opsBSriUO_C)FrBQwZDjDsOLYz)qYK>F&ZrQE5tk!UxI`?=xC1;Ea@ zmwgnw7bn|iRfL2_D&$V=Avo!P%%Px}>#i^H(9(RB_tFwDwS*KGu%>MUr|+d#G>BWC ztH^xR4^T!l&pAnNV`|9y@IV>Z<@77FeQf@K7?@TBx$6mr?j#s3tMY+|ss`xV!fpd> z7_?4r9t!CO@AZX2^VeX~XzBGNx%TCU>aqP*SMe+L2u8$9`X?c+#K%&y{JlEvawn~G znFZvfk$KgGG;e}57x^xEXAD_u;TZO#7AvEuU+|RM*$-lWojx36C_~Wr=`oXSA?EA@ z1y`5*>>mF)AF)u2prn0ZaAqy$cAV7Tnw*Q67s@7EY&~ZUIQI~3QbQ9yYfMmPtqKIP()Z%or9>k_*FB%!G6(#FWyILMot7{eqydbURI^E%R%&5t>AwS zi7!8=S$bTP%TR4-{+zCMycSmI*p3&- zWDEHF$O*BmF)WM6iG%z{f|m}u!5u~nbTlCY9@t+!o|fXC(m9a9I=VOoUjDtKC@UpI2HLXDmjF`uPK~oHLIHv6*1$zksu!vc*Sxp{I3#ds~KF}FIYrym?C^tlkhVAo<=5?9< z_AY1EE(zGIgJ9_A3y@YcO?{t^_QmV#Wj~vg_P9QCMWIm@AxqUjW8@)nZbT|;w&lrjG;Tj1qtB@k%X9t{)pL0H=8>jL`0}5T_5R=` zdH!6`Xu9PQ$BY~2(DnjKxJNy9e0qLX>*iM6C)?IbM{qfE#aimB*s z$qE>`QBM6|Nd_5J0h}lZHyH|MeG?T;ga{+m5NYZhr+|P-w&)q)jgzqXGq^ zPwQOGiF+>4p2=G%q^zQcJL#ZyFUS{~pE#xVQjsD}-#8o4|IFL-DY;V}G)dW*-g;hw zHY=(kWVYn=dP>6FI%FXW_BI4+MS3lnRghzVc&(^AL6jSi!*p8j5x#`%Wf0d?)QlZ1 zez$E`vEq{3>$!XoT6`W>bNKrPEQBIH`$tx?O?%0>=%yofKvH_Qc7pC~dy-{o=@T_q zFC6SQBF2ZVRi_`UZJTO#b&;Pt(b_QZGSsl{N1VJORl1Thh2xAgf6Eric(l%$_oOe9 zLKDf0Sl}sh1c5eZ5f=9H%SBx^3C=Z+R*aO&o3Xda{;hzPXDAf#Uizow%f2ulFUEZ& zHRK@BV1DU!SXrUdjpc&7R+!38)}1KSFzVU5ZJ@@NnzD}1R~c!+IpIzDk;VfvPih;h zrFW|N2%7Bv9=rl2E8sua6)?M@wxL`%5ZP~=S6he4FBM=~PQ7*3RQV*}T4Ll*+5HcB zZi;;y^KXjY$`F);h8I_Vo1C^I1z}lj&^fS^&u+z24!@gtv*-35GTsoLX#!YtKPMDt z{T`vBW>7P1q*ny*^anS?PLg?}DdA>NqDR4_K^aZ<$5KiX`9;CC-6|@Xm@@pZgI%!N zo}Nh!9aHyAm0hapiXW_Mb`_ytXH(eQRIfv_%}n0PU|ZraT^+Ny?{XByY)kc;6&hrejdz34GL|xt6p!$P1rLujN5SQxSZd*mTJ@F%RQ?IeV z?t4cmfth+OD%S$)j=PTadq!VTPGHB1wYO((YjUclMzN!h#~c0P|D@9>PFJc0RI>cs zuQdg{|DjR>bH6sNrd;`Om(_v0tSQ`O-C-@832Rwc%Yw-a1_m%NP}hKg0SpXuU|?Vh t0|R#$7-Yi000suoIv?IK_=5ohcQc1ROv=}<$SUCT{^aDx_l}2U{}=A!xwrrT diff --git a/docs/reference/connector/docs/images/connectors-overview.svg b/docs/reference/connector/docs/images/connectors-overview.svg new file mode 100644 index 0000000000000..0a7fb30c61d6d --- /dev/null +++ b/docs/reference/connector/docs/images/connectors-overview.svg @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/reference/connector/docs/index.asciidoc b/docs/reference/connector/docs/index.asciidoc index 481e124a1a117..dfca45f86ebce 100644 --- a/docs/reference/connector/docs/index.asciidoc +++ b/docs/reference/connector/docs/index.asciidoc @@ -72,7 +72,7 @@ Refer to <> for details. The following diagram provides a high-level overview of the Elastic connectors offering and some key facts. -image::connectors-overview.png[align="center",width="100%"] +image::connectors-overview.svg[align="center",width="100%"] [discrete#es-connectors-overview-available-connectors] == Available connectors and feature support From bc25a73543d1f7b0e523e96dd81786147d2dd0b3 Mon Sep 17 00:00:00 2001 From: Pete Gillin Date: Mon, 9 Dec 2024 14:28:24 +0000 Subject: [PATCH 125/204] Update `UpdateForV9` in `AttachmentProcessor` (#118186) We are not going to make this change in V9. We may do it in V10. This change just bumps the annotation to remind us to revisit. Since we are living with this for a while, it seems worth improving the documentation. This now encourages explicitly setting the option one way or the other, since you get a warning if you omit it. It also changes the existing examples to use true rather than false, as that's our recommendation. And it adds a new section with an example where it's true, and moves the content previously in a note into that section. --- .../ingest/processors/attachment.asciidoc | 91 ++++++++++++++----- .../attachment/AttachmentProcessor.java | 10 +- 2 files changed, 73 insertions(+), 28 deletions(-) diff --git a/docs/reference/ingest/processors/attachment.asciidoc b/docs/reference/ingest/processors/attachment.asciidoc index fd2866906c1d0..bd5b8db562ae2 100644 --- a/docs/reference/ingest/processors/attachment.asciidoc +++ b/docs/reference/ingest/processors/attachment.asciidoc @@ -19,15 +19,15 @@ representation. The processor will skip the base64 decoding then. .Attachment options [options="header"] |====== -| Name | Required | Default | Description -| `field` | yes | - | The field to get the base64 encoded field from -| `target_field` | no | attachment | The field that will hold the attachment information -| `indexed_chars` | no | 100000 | The number of chars being used for extraction to prevent huge fields. Use `-1` for no limit. -| `indexed_chars_field` | no | `null` | Field name from which you can overwrite the number of chars being used for extraction. See `indexed_chars`. -| `properties` | no | all properties | Array of properties to select to be stored. Can be `content`, `title`, `name`, `author`, `keywords`, `date`, `content_type`, `content_length`, `language` -| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document -| `remove_binary` | no | `false` | If `true`, the binary `field` will be removed from the document -| `resource_name` | no | | Field containing the name of the resource to decode. If specified, the processor passes this resource name to the underlying Tika library to enable https://tika.apache.org/1.24.1/detection.html#Resource_Name_Based_Detection[Resource Name Based Detection]. +| Name | Required | Default | Description +| `field` | yes | - | The field to get the base64 encoded field from +| `target_field` | no | attachment | The field that will hold the attachment information +| `indexed_chars` | no | 100000 | The number of chars being used for extraction to prevent huge fields. Use `-1` for no limit. +| `indexed_chars_field` | no | `null` | Field name from which you can overwrite the number of chars being used for extraction. See `indexed_chars`. +| `properties` | no | all properties | Array of properties to select to be stored. Can be `content`, `title`, `name`, `author`, `keywords`, `date`, `content_type`, `content_length`, `language` +| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document +| `remove_binary` | encouraged | `false` | If `true`, the binary `field` will be removed from the document. This option is not required, but setting it explicitly is encouraged, and omitting it will result in a warning. +| `resource_name` | no | | Field containing the name of the resource to decode. If specified, the processor passes this resource name to the underlying Tika library to enable https://tika.apache.org/1.24.1/detection.html#Resource_Name_Based_Detection[Resource Name Based Detection]. |====== [discrete] @@ -58,7 +58,7 @@ PUT _ingest/pipeline/attachment { "attachment" : { "field" : "data", - "remove_binary": false + "remove_binary": true } } ] @@ -82,7 +82,6 @@ The document's `attachment` object contains extracted properties for the file: "_seq_no": 22, "_primary_term": 1, "_source": { - "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=", "attachment": { "content_type": "application/rtf", "language": "ro", @@ -94,9 +93,6 @@ The document's `attachment` object contains extracted properties for the file: ---- // TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/] -NOTE: Keeping the binary as a field within the document might consume a lot of resources. It is highly recommended - to remove that field from the document. Set `remove_binary` to `true` to automatically remove the field. - [[attachment-fields]] ==== Exported fields @@ -143,7 +139,7 @@ PUT _ingest/pipeline/attachment "attachment" : { "field" : "data", "properties": [ "content", "title" ], - "remove_binary": false + "remove_binary": true } } ] @@ -154,6 +150,59 @@ NOTE: Extracting contents from binary data is a resource intensive operation and consumes a lot of resources. It is highly recommended to run pipelines using this processor in a dedicated ingest node. +[[attachment-keep-binary]] +==== Keeping the attachment binary + +Keeping the binary as a field within the document might consume a lot of resources. It is highly recommended to remove +that field from the document, by setting `remove_binary` to `true` to automatically remove the field, as in the other +examples shown on this page. If you _do_ want to keep the binary field, explicitly set `remove_binary` to `false` to +avoid the warning you get from omitting it: + +[source,console] +---- +PUT _ingest/pipeline/attachment +{ + "description" : "Extract attachment information including original binary", + "processors" : [ + { + "attachment" : { + "field" : "data", + "remove_binary": false + } + } + ] +} +PUT my-index-000001/_doc/my_id?pipeline=attachment +{ + "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=" +} +GET my-index-000001/_doc/my_id +---- + +The document's `_source` object includes the original binary field: + +[source,console-result] +---- +{ + "found": true, + "_index": "my-index-000001", + "_id": "my_id", + "_version": 1, + "_seq_no": 22, + "_primary_term": 1, + "_source": { + "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=", + "attachment": { + "content_type": "application/rtf", + "language": "ro", + "content": "Lorem ipsum dolor sit amet", + "content_length": 28 + } + } +} +---- +// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/] + [[attachment-cbor]] ==== Use the attachment processor with CBOR @@ -170,7 +219,7 @@ PUT _ingest/pipeline/cbor-attachment { "attachment" : { "field" : "data", - "remove_binary": false + "remove_binary": true } } ] @@ -226,7 +275,7 @@ PUT _ingest/pipeline/attachment "field" : "data", "indexed_chars" : 11, "indexed_chars_field" : "max_size", - "remove_binary": false + "remove_binary": true } } ] @@ -250,7 +299,6 @@ Returns this: "_seq_no": 35, "_primary_term": 1, "_source": { - "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=", "attachment": { "content_type": "application/rtf", "language": "is", @@ -274,7 +322,7 @@ PUT _ingest/pipeline/attachment "field" : "data", "indexed_chars" : 11, "indexed_chars_field" : "max_size", - "remove_binary": false + "remove_binary": true } } ] @@ -299,7 +347,6 @@ Returns this: "_seq_no": 40, "_primary_term": 1, "_source": { - "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=", "max_size": 5, "attachment": { "content_type": "application/rtf", @@ -358,7 +405,7 @@ PUT _ingest/pipeline/attachment "attachment": { "target_field": "_ingest._value.attachment", "field": "_ingest._value.data", - "remove_binary": false + "remove_binary": true } } } @@ -396,7 +443,6 @@ Returns this: "attachments" : [ { "filename" : "ipsum.txt", - "data" : "dGhpcyBpcwpqdXN0IHNvbWUgdGV4dAo=", "attachment" : { "content_type" : "text/plain; charset=ISO-8859-1", "language" : "en", @@ -406,7 +452,6 @@ Returns this: }, { "filename" : "test.txt", - "data" : "VGhpcyBpcyBhIHRlc3QK", "attachment" : { "content_type" : "text/plain; charset=ISO-8859-1", "language" : "en", diff --git a/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java index 007fe39d72e61..83a7bdf7e224a 100644 --- a/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java +++ b/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java @@ -18,7 +18,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.UpdateForV9; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; @@ -196,7 +196,7 @@ public IngestDocument execute(IngestDocument ingestDocument) { * @param property property to add * @param value value to add */ - private void addAdditionalField(Map additionalFields, Property property, String value) { + private void addAdditionalField(Map additionalFields, Property property, String value) { if (properties.contains(property) && Strings.hasLength(value)) { additionalFields.put(property.toLowerCase(), value); } @@ -233,7 +233,7 @@ public AttachmentProcessor create( String processorTag, String description, Map config - ) throws Exception { + ) { String field = readStringProperty(TYPE, processorTag, config, "field"); String resourceName = readOptionalStringProperty(TYPE, processorTag, config, "resource_name"); String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "attachment"); @@ -241,8 +241,8 @@ public AttachmentProcessor create( int indexedChars = readIntProperty(TYPE, processorTag, config, "indexed_chars", NUMBER_OF_CHARS_INDEXED); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); String indexedCharsField = readOptionalStringProperty(TYPE, processorTag, config, "indexed_chars_field"); - @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT) - // update the [remove_binary] default to be 'true' assuming enough time has passed. Deprecated in September 2022. + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) + // Revisit whether we want to update the [remove_binary] default to be 'true' - would need to find a way to do this safely Boolean removeBinary = readOptionalBooleanProperty(TYPE, processorTag, config, "remove_binary"); if (removeBinary == null) { DEPRECATION_LOGGER.warn( From 527c3e3041d7dc3098799677878a5887809fcbdd Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 10 Dec 2024 01:34:07 +1100 Subject: [PATCH 126/204] Mute org.elasticsearch.xpack.test.rest.XPackRestIT test {p0=migrate/10_reindex/Test Reindex With Bad Data Stream Name} #118272 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index eecb7ac3d7e59..edb48e0cc3a7e 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -288,6 +288,9 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveTests method: test60StartAndStop issue: https://github.com/elastic/elasticsearch/issues/118216 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=migrate/10_reindex/Test Reindex With Bad Data Stream Name} + issue: https://github.com/elastic/elasticsearch/issues/118272 # Examples: # From e9f507464c729fbcb8fa0fc57948c163bee1a125 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 10 Dec 2024 01:34:24 +1100 Subject: [PATCH 127/204] Mute org.elasticsearch.xpack.test.rest.XPackRestIT test {p0=migrate/10_reindex/Test Reindex With Unsupported Mode} #118273 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index edb48e0cc3a7e..ecb3c0293d630 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -291,6 +291,9 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=migrate/10_reindex/Test Reindex With Bad Data Stream Name} issue: https://github.com/elastic/elasticsearch/issues/118272 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=migrate/10_reindex/Test Reindex With Unsupported Mode} + issue: https://github.com/elastic/elasticsearch/issues/118273 # Examples: # From e593eb377f98975f95eef4fa6a1c1caae1238d48 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 10 Dec 2024 01:34:41 +1100 Subject: [PATCH 128/204] Mute org.elasticsearch.xpack.test.rest.XPackRestIT test {p0=migrate/10_reindex/Test Reindex With Nonexistent Data Stream} #118274 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index ecb3c0293d630..f922a1a27b629 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -294,6 +294,9 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=migrate/10_reindex/Test Reindex With Unsupported Mode} issue: https://github.com/elastic/elasticsearch/issues/118273 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=migrate/10_reindex/Test Reindex With Nonexistent Data Stream} + issue: https://github.com/elastic/elasticsearch/issues/118274 # Examples: # From f40dc99f9101e97f4100df2a63e73757be52ffa5 Mon Sep 17 00:00:00 2001 From: Dan Rubinstein Date: Mon, 9 Dec 2024 10:14:32 -0500 Subject: [PATCH 129/204] Adding transforms migration guide for 9.0 (#117353) * Adding transforms migration guide for 9.0 * Adding shared transform attribute and simplifying wording --------- Co-authored-by: Elastic Machine --- .../migrate_9_0/transforms-migration-guide.asciidoc | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/reference/migration/migrate_9_0/transforms-migration-guide.asciidoc diff --git a/docs/reference/migration/migrate_9_0/transforms-migration-guide.asciidoc b/docs/reference/migration/migrate_9_0/transforms-migration-guide.asciidoc new file mode 100644 index 0000000000000..d41c524d68d5c --- /dev/null +++ b/docs/reference/migration/migrate_9_0/transforms-migration-guide.asciidoc @@ -0,0 +1,9 @@ +[[transforms-migration-guide]] +== {transforms-cap} migration guide +This migration guide helps you upgrade your {transforms} to work with the 9.0 release. Each section outlines a breaking change and any manual steps needed to upgrade your {transforms} to be compatible with 9.0. + + +=== Updating deprecated {transform} roles (`data_frame_transforms_admin` and `data_frame_transforms_user`) +If you have existing {transforms} that use deprecated {transform} roles (`data_frame_transforms_admin` or `data_frame_transforms_user`) you must update them to use the new equivalent {transform} roles (`transform_admin` or `transform_user`). To update your {transform} roles: +1. Switch to a user with the `transform_admin` role (to replace `data_frame_transforms_admin`) or the `transform_user` role (to replace `data_frame_transforms_user`). +2. Call the <> with that user. From 2ecf981f24ca5d480466a4fcc669aeb52d063657 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 9 Dec 2024 15:22:48 +0000 Subject: [PATCH 130/204] [ML] Refactor the Chunker classes to return offsets (#117977) --- .../xpack/inference/chunking/Chunker.java | 4 +- .../chunking/EmbeddingRequestChunker.java | 55 ++++++++++++------- .../chunking/SentenceBoundaryChunker.java | 20 ++++--- .../chunking/WordBoundaryChunker.java | 22 +++----- .../EmbeddingRequestChunkerTests.java | 24 ++++---- .../SentenceBoundaryChunkerTests.java | 34 +++++++++--- .../chunking/WordBoundaryChunkerTests.java | 36 ++++++++---- 7 files changed, 119 insertions(+), 76 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/Chunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/Chunker.java index af7c706c807ec..b8908ee139c29 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/Chunker.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/Chunker.java @@ -12,5 +12,7 @@ import java.util.List; public interface Chunker { - List chunk(String input, ChunkingSettings chunkingSettings); + record ChunkOffset(int start, int end) {}; + + List chunk(String input, ChunkingSettings chunkingSettings); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java index c5897f32d6eb8..2aef54e56f4b9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java @@ -68,7 +68,7 @@ public static EmbeddingType fromDenseVectorElementType(DenseVectorFieldMapper.El private final EmbeddingType embeddingType; private final ChunkingSettings chunkingSettings; - private List> chunkedInputs; + private List chunkedOffsets; private List>> floatResults; private List>> byteResults; private List>> sparseResults; @@ -109,7 +109,7 @@ public EmbeddingRequestChunker( } private void splitIntoBatchedRequests(List inputs) { - Function> chunkFunction; + Function> chunkFunction; if (chunkingSettings != null) { var chunker = ChunkerBuilder.fromChunkingStrategy(chunkingSettings.getChunkingStrategy()); chunkFunction = input -> chunker.chunk(input, chunkingSettings); @@ -118,7 +118,7 @@ private void splitIntoBatchedRequests(List inputs) { chunkFunction = input -> chunker.chunk(input, wordsPerChunk, chunkOverlap); } - chunkedInputs = new ArrayList<>(inputs.size()); + chunkedOffsets = new ArrayList<>(inputs.size()); switch (embeddingType) { case FLOAT -> floatResults = new ArrayList<>(inputs.size()); case BYTE -> byteResults = new ArrayList<>(inputs.size()); @@ -128,18 +128,19 @@ private void splitIntoBatchedRequests(List inputs) { for (int i = 0; i < inputs.size(); i++) { var chunks = chunkFunction.apply(inputs.get(i)); - int numberOfSubBatches = addToBatches(chunks, i); + var offSetsAndInput = new ChunkOffsetsAndInput(chunks, inputs.get(i)); + int numberOfSubBatches = addToBatches(offSetsAndInput, i); // size the results array with the expected number of request/responses switch (embeddingType) { case FLOAT -> floatResults.add(new AtomicArray<>(numberOfSubBatches)); case BYTE -> byteResults.add(new AtomicArray<>(numberOfSubBatches)); case SPARSE -> sparseResults.add(new AtomicArray<>(numberOfSubBatches)); } - chunkedInputs.add(chunks); + chunkedOffsets.add(offSetsAndInput); } } - private int addToBatches(List chunks, int inputIndex) { + private int addToBatches(ChunkOffsetsAndInput chunk, int inputIndex) { BatchRequest lastBatch; if (batchedRequests.isEmpty()) { lastBatch = new BatchRequest(new ArrayList<>()); @@ -157,16 +158,24 @@ private int addToBatches(List chunks, int inputIndex) { if (freeSpace > 0) { // use any free space in the previous batch before creating new batches - int toAdd = Math.min(freeSpace, chunks.size()); - lastBatch.addSubBatch(new SubBatch(chunks.subList(0, toAdd), new SubBatchPositionsAndCount(inputIndex, chunkIndex++, toAdd))); + int toAdd = Math.min(freeSpace, chunk.offsets().size()); + lastBatch.addSubBatch( + new SubBatch( + new ChunkOffsetsAndInput(chunk.offsets().subList(0, toAdd), chunk.input()), + new SubBatchPositionsAndCount(inputIndex, chunkIndex++, toAdd) + ) + ); } int start = freeSpace; - while (start < chunks.size()) { - int toAdd = Math.min(maxNumberOfInputsPerBatch, chunks.size() - start); + while (start < chunk.offsets().size()) { + int toAdd = Math.min(maxNumberOfInputsPerBatch, chunk.offsets().size() - start); var batch = new BatchRequest(new ArrayList<>()); batch.addSubBatch( - new SubBatch(chunks.subList(start, start + toAdd), new SubBatchPositionsAndCount(inputIndex, chunkIndex++, toAdd)) + new SubBatch( + new ChunkOffsetsAndInput(chunk.offsets().subList(start, start + toAdd), chunk.input()), + new SubBatchPositionsAndCount(inputIndex, chunkIndex++, toAdd) + ) ); batchedRequests.add(batch); start += toAdd; @@ -333,8 +342,8 @@ public void onFailure(Exception e) { } private void sendResponse() { - var response = new ArrayList(chunkedInputs.size()); - for (int i = 0; i < chunkedInputs.size(); i++) { + var response = new ArrayList(chunkedOffsets.size()); + for (int i = 0; i < chunkedOffsets.size(); i++) { if (errors.get(i) != null) { response.add(errors.get(i)); } else { @@ -348,9 +357,9 @@ private void sendResponse() { private ChunkedInferenceServiceResults mergeResultsWithInputs(int resultIndex) { return switch (embeddingType) { - case FLOAT -> mergeFloatResultsWithInputs(chunkedInputs.get(resultIndex), floatResults.get(resultIndex)); - case BYTE -> mergeByteResultsWithInputs(chunkedInputs.get(resultIndex), byteResults.get(resultIndex)); - case SPARSE -> mergeSparseResultsWithInputs(chunkedInputs.get(resultIndex), sparseResults.get(resultIndex)); + case FLOAT -> mergeFloatResultsWithInputs(chunkedOffsets.get(resultIndex).toChunkText(), floatResults.get(resultIndex)); + case BYTE -> mergeByteResultsWithInputs(chunkedOffsets.get(resultIndex).toChunkText(), byteResults.get(resultIndex)); + case SPARSE -> mergeSparseResultsWithInputs(chunkedOffsets.get(resultIndex).toChunkText(), sparseResults.get(resultIndex)); }; } @@ -428,7 +437,7 @@ public void addSubBatch(SubBatch sb) { } public List inputs() { - return subBatches.stream().flatMap(s -> s.requests().stream()).collect(Collectors.toList()); + return subBatches.stream().flatMap(s -> s.requests().toChunkText().stream()).collect(Collectors.toList()); } } @@ -441,9 +450,15 @@ public record BatchRequestAndListener(BatchRequest batch, ActionListener requests, SubBatchPositionsAndCount positions) { - public int size() { - return requests.size(); + record SubBatch(ChunkOffsetsAndInput requests, SubBatchPositionsAndCount positions) { + int size() { + return requests.offsets().size(); + } + } + + record ChunkOffsetsAndInput(List offsets, String input) { + List toChunkText() { + return offsets.stream().map(o -> input.substring(o.start(), o.end())).collect(Collectors.toList()); } } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunker.java index 5df940d6a3fba..b2d6c83b89211 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunker.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunker.java @@ -34,7 +34,6 @@ public class SentenceBoundaryChunker implements Chunker { public SentenceBoundaryChunker() { sentenceIterator = BreakIterator.getSentenceInstance(Locale.ROOT); wordIterator = BreakIterator.getWordInstance(Locale.ROOT); - } /** @@ -45,7 +44,7 @@ public SentenceBoundaryChunker() { * @return The input text chunked */ @Override - public List chunk(String input, ChunkingSettings chunkingSettings) { + public List chunk(String input, ChunkingSettings chunkingSettings) { if (chunkingSettings instanceof SentenceBoundaryChunkingSettings sentenceBoundaryChunkingSettings) { return chunk(input, sentenceBoundaryChunkingSettings.maxChunkSize, sentenceBoundaryChunkingSettings.sentenceOverlap > 0); } else { @@ -65,8 +64,8 @@ public List chunk(String input, ChunkingSettings chunkingSettings) { * @param maxNumberWordsPerChunk Maximum size of the chunk * @return The input text chunked */ - public List chunk(String input, int maxNumberWordsPerChunk, boolean includePrecedingSentence) { - var chunks = new ArrayList(); + public List chunk(String input, int maxNumberWordsPerChunk, boolean includePrecedingSentence) { + var chunks = new ArrayList(); sentenceIterator.setText(input); wordIterator.setText(input); @@ -91,7 +90,7 @@ public List chunk(String input, int maxNumberWordsPerChunk, boolean incl int nextChunkWordCount = wordsInSentenceCount; if (chunkWordCount > 0) { // add a new chunk containing all the input up to this sentence - chunks.add(input.substring(chunkStart, chunkEnd)); + chunks.add(new ChunkOffset(chunkStart, chunkEnd)); if (includePrecedingSentence) { if (wordsInPrecedingSentenceCount + wordsInSentenceCount > maxNumberWordsPerChunk) { @@ -127,12 +126,17 @@ public List chunk(String input, int maxNumberWordsPerChunk, boolean incl for (; i < sentenceSplits.size() - 1; i++) { // Because the substring was passed to splitLongSentence() // the returned positions need to be offset by chunkStart - chunks.add(input.substring(chunkStart + sentenceSplits.get(i).start(), chunkStart + sentenceSplits.get(i).end())); + chunks.add( + new ChunkOffset( + chunkStart + sentenceSplits.get(i).offsets().start(), + chunkStart + sentenceSplits.get(i).offsets().end() + ) + ); } // The final split is partially filled. // Set the next chunk start to the beginning of the // final split of the long sentence. - chunkStart = chunkStart + sentenceSplits.get(i).start(); // start pos needs to be offset by chunkStart + chunkStart = chunkStart + sentenceSplits.get(i).offsets().start(); // start pos needs to be offset by chunkStart chunkWordCount = sentenceSplits.get(i).wordCount(); } } else { @@ -151,7 +155,7 @@ public List chunk(String input, int maxNumberWordsPerChunk, boolean incl } if (chunkWordCount > 0) { - chunks.add(input.substring(chunkStart)); + chunks.add(new ChunkOffset(chunkStart, input.length())); } return chunks; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunker.java index c9c752b9aabbc..b15e2134f4cf7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunker.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunker.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.stream.Collectors; /** * Breaks text into smaller strings or chunks on Word boundaries. @@ -35,7 +36,7 @@ public WordBoundaryChunker() { wordIterator = BreakIterator.getWordInstance(Locale.ROOT); } - record ChunkPosition(int start, int end, int wordCount) {} + record ChunkPosition(ChunkOffset offsets, int wordCount) {} /** * Break the input text into small chunks as dictated @@ -45,7 +46,7 @@ record ChunkPosition(int start, int end, int wordCount) {} * @return List of chunked text */ @Override - public List chunk(String input, ChunkingSettings chunkingSettings) { + public List chunk(String input, ChunkingSettings chunkingSettings) { if (chunkingSettings instanceof WordBoundaryChunkingSettings wordBoundaryChunkerSettings) { return chunk(input, wordBoundaryChunkerSettings.maxChunkSize, wordBoundaryChunkerSettings.overlap); } else { @@ -64,18 +65,9 @@ public List chunk(String input, ChunkingSettings chunkingSettings) { * Can be 0 but must be non-negative. * @return List of chunked text */ - public List chunk(String input, int chunkSize, int overlap) { - - if (input.isEmpty()) { - return List.of(""); - } - + public List chunk(String input, int chunkSize, int overlap) { var chunkPositions = chunkPositions(input, chunkSize, overlap); - var chunks = new ArrayList(chunkPositions.size()); - for (var pos : chunkPositions) { - chunks.add(input.substring(pos.start, pos.end)); - } - return chunks; + return chunkPositions.stream().map(ChunkPosition::offsets).collect(Collectors.toList()); } /** @@ -127,7 +119,7 @@ List chunkPositions(String input, int chunkSize, int overlap) { wordsSinceStartWindowWasMarked++; if (wordsInChunkCountIncludingOverlap >= chunkSize) { - chunkPositions.add(new ChunkPosition(windowStart, boundary, wordsInChunkCountIncludingOverlap)); + chunkPositions.add(new ChunkPosition(new ChunkOffset(windowStart, boundary), wordsInChunkCountIncludingOverlap)); wordsInChunkCountIncludingOverlap = overlap; if (overlap == 0) { @@ -149,7 +141,7 @@ List chunkPositions(String input, int chunkSize, int overlap) { // if it ends on a boundary than the count should equal overlap in which case // we can ignore it, unless this is the first chunk in which case we want to add it if (wordsInChunkCountIncludingOverlap > overlap || chunkPositions.isEmpty()) { - chunkPositions.add(new ChunkPosition(windowStart, input.length(), wordsInChunkCountIncludingOverlap)); + chunkPositions.add(new ChunkPosition(new ChunkOffset(windowStart, input.length()), wordsInChunkCountIncludingOverlap)); } return chunkPositions; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java index 4fdf254101d3e..a82d2f474ca4a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java @@ -62,7 +62,7 @@ public void testMultipleShortInputsAreSingleBatch() { var subBatches = batches.get(0).batch().subBatches(); for (int i = 0; i < inputs.size(); i++) { var subBatch = subBatches.get(i); - assertThat(subBatch.requests(), contains(inputs.get(i))); + assertThat(subBatch.requests().toChunkText(), contains(inputs.get(i))); assertEquals(0, subBatch.positions().chunkIndex()); assertEquals(i, subBatch.positions().inputIndex()); assertEquals(1, subBatch.positions().embeddingCount()); @@ -102,7 +102,7 @@ public void testManyInputsMakeManyBatches() { var subBatches = batches.get(0).batch().subBatches(); for (int i = 0; i < batches.size(); i++) { var subBatch = subBatches.get(i); - assertThat(subBatch.requests(), contains(inputs.get(i))); + assertThat(subBatch.requests().toChunkText(), contains(inputs.get(i))); assertEquals(0, subBatch.positions().chunkIndex()); assertEquals(inputIndex, subBatch.positions().inputIndex()); assertEquals(1, subBatch.positions().embeddingCount()); @@ -146,7 +146,7 @@ public void testChunkingSettingsProvided() { var subBatches = batches.get(0).batch().subBatches(); for (int i = 0; i < batches.size(); i++) { var subBatch = subBatches.get(i); - assertThat(subBatch.requests(), contains(inputs.get(i))); + assertThat(subBatch.requests().toChunkText(), contains(inputs.get(i))); assertEquals(0, subBatch.positions().chunkIndex()); assertEquals(inputIndex, subBatch.positions().inputIndex()); assertEquals(1, subBatch.positions().embeddingCount()); @@ -184,17 +184,17 @@ public void testLongInputChunkedOverMultipleBatches() { assertEquals(0, subBatch.positions().inputIndex()); assertEquals(0, subBatch.positions().chunkIndex()); assertEquals(1, subBatch.positions().embeddingCount()); - assertThat(subBatch.requests(), contains("1st small")); + assertThat(subBatch.requests().toChunkText(), contains("1st small")); } { var subBatch = batch.subBatches().get(1); assertEquals(1, subBatch.positions().inputIndex()); // 2nd input assertEquals(0, subBatch.positions().chunkIndex()); // 1st part of the 2nd input assertEquals(4, subBatch.positions().embeddingCount()); // 4 chunks - assertThat(subBatch.requests().get(0), startsWith("passage_input0 ")); - assertThat(subBatch.requests().get(1), startsWith(" passage_input20 ")); - assertThat(subBatch.requests().get(2), startsWith(" passage_input40 ")); - assertThat(subBatch.requests().get(3), startsWith(" passage_input60 ")); + assertThat(subBatch.requests().toChunkText().get(0), startsWith("passage_input0 ")); + assertThat(subBatch.requests().toChunkText().get(1), startsWith(" passage_input20 ")); + assertThat(subBatch.requests().toChunkText().get(2), startsWith(" passage_input40 ")); + assertThat(subBatch.requests().toChunkText().get(3), startsWith(" passage_input60 ")); } } { @@ -207,22 +207,22 @@ public void testLongInputChunkedOverMultipleBatches() { assertEquals(1, subBatch.positions().inputIndex()); // 2nd input assertEquals(1, subBatch.positions().chunkIndex()); // 2nd part of the 2nd input assertEquals(2, subBatch.positions().embeddingCount()); - assertThat(subBatch.requests().get(0), startsWith(" passage_input80 ")); - assertThat(subBatch.requests().get(1), startsWith(" passage_input100 ")); + assertThat(subBatch.requests().toChunkText().get(0), startsWith(" passage_input80 ")); + assertThat(subBatch.requests().toChunkText().get(1), startsWith(" passage_input100 ")); } { var subBatch = batch.subBatches().get(1); assertEquals(2, subBatch.positions().inputIndex()); // 3rd input assertEquals(0, subBatch.positions().chunkIndex()); // 1st and only part assertEquals(1, subBatch.positions().embeddingCount()); // 1 chunk - assertThat(subBatch.requests(), contains("2nd small")); + assertThat(subBatch.requests().toChunkText(), contains("2nd small")); } { var subBatch = batch.subBatches().get(2); assertEquals(3, subBatch.positions().inputIndex()); // 4th input assertEquals(0, subBatch.positions().chunkIndex()); // 1st and only part assertEquals(1, subBatch.positions().embeddingCount()); // 1 chunk - assertThat(subBatch.requests(), contains("3rd small")); + assertThat(subBatch.requests().toChunkText(), contains("3rd small")); } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java index afce8c57e0350..de943f7f57ab8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java @@ -15,7 +15,9 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Locale; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.inference.chunking.WordBoundaryChunkerTests.TEST_TEXT; import static org.hamcrest.Matchers.containsString; @@ -27,10 +29,24 @@ public class SentenceBoundaryChunkerTests extends ESTestCase { + /** + * Utility method for testing. + * Use the chunk functions that return offsets where possible + */ + private List textChunks( + SentenceBoundaryChunker chunker, + String input, + int maxNumberWordsPerChunk, + boolean includePrecedingSentence + ) { + var chunkPositions = chunker.chunk(input, maxNumberWordsPerChunk, includePrecedingSentence); + return chunkPositions.stream().map(offset -> input.substring(offset.start(), offset.end())).collect(Collectors.toList()); + } + public void testChunkSplitLargeChunkSizes() { for (int maxWordsPerChunk : new int[] { 100, 200 }) { var chunker = new SentenceBoundaryChunker(); - var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk, false); + var chunks = textChunks(chunker, TEST_TEXT, maxWordsPerChunk, false); int numChunks = expectedNumberOfChunks(sentenceSizes(TEST_TEXT), maxWordsPerChunk); assertThat("words per chunk " + maxWordsPerChunk, chunks, hasSize(numChunks)); @@ -48,7 +64,7 @@ public void testChunkSplitLargeChunkSizes_withOverlap() { boolean overlap = true; for (int maxWordsPerChunk : new int[] { 70, 80, 100, 120, 150, 200 }) { var chunker = new SentenceBoundaryChunker(); - var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk, overlap); + var chunks = textChunks(chunker, TEST_TEXT, maxWordsPerChunk, overlap); int[] overlaps = chunkOverlaps(sentenceSizes(TEST_TEXT), maxWordsPerChunk, overlap); assertThat("words per chunk " + maxWordsPerChunk, chunks, hasSize(overlaps.length)); @@ -107,7 +123,7 @@ public void testWithOverlap_SentencesFitInChunks() { } var chunker = new SentenceBoundaryChunker(); - var chunks = chunker.chunk(sb.toString(), chunkSize, true); + var chunks = textChunks(chunker, sb.toString(), chunkSize, true); assertThat(chunks, hasSize(numChunks)); for (int i = 0; i < numChunks; i++) { assertThat("num sentences " + numSentences, chunks.get(i), startsWith("SStart" + sentenceStartIndexes[i])); @@ -128,10 +144,10 @@ private String makeSentence(int numWords, int sentenceIndex) { public void testChunk_ChunkSizeLargerThanText() { int maxWordsPerChunk = 500; var chunker = new SentenceBoundaryChunker(); - var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk, false); + var chunks = textChunks(chunker, TEST_TEXT, maxWordsPerChunk, false); assertEquals(chunks.get(0), TEST_TEXT); - chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk, true); + chunks = textChunks(chunker, TEST_TEXT, maxWordsPerChunk, true); assertEquals(chunks.get(0), TEST_TEXT); } @@ -142,7 +158,7 @@ public void testChunkSplit_SentencesLongerThanChunkSize() { for (int i = 0; i < chunkSizes.length; i++) { int maxWordsPerChunk = chunkSizes[i]; var chunker = new SentenceBoundaryChunker(); - var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk, false); + var chunks = textChunks(chunker, TEST_TEXT, maxWordsPerChunk, false); assertThat("words per chunk " + maxWordsPerChunk, chunks, hasSize(expectedNumberOFChunks[i])); for (var chunk : chunks) { @@ -171,7 +187,7 @@ public void testChunkSplit_SentencesLongerThanChunkSize_WithOverlap() { for (int i = 0; i < chunkSizes.length; i++) { int maxWordsPerChunk = chunkSizes[i]; var chunker = new SentenceBoundaryChunker(); - var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk, true); + var chunks = textChunks(chunker, TEST_TEXT, maxWordsPerChunk, true); assertThat(chunks.get(0), containsString("Word segmentation is the problem of dividing")); assertThat(chunks.get(chunks.size() - 1), containsString(", with solidification being a stronger norm.")); } @@ -190,7 +206,7 @@ public void testShortLongShortSentences_WithOverlap() { } var chunker = new SentenceBoundaryChunker(); - var chunks = chunker.chunk(sb.toString(), maxWordsPerChunk, true); + var chunks = textChunks(chunker, sb.toString(), maxWordsPerChunk, true); assertThat(chunks, hasSize(5)); assertTrue(chunks.get(0).trim().startsWith("SStart0")); // Entire sentence assertTrue(chunks.get(0).trim().endsWith(".")); // Entire sentence @@ -303,7 +319,7 @@ public void testChunkSplitLargeChunkSizesWithChunkingSettings() { for (int maxWordsPerChunk : new int[] { 100, 200 }) { var chunker = new SentenceBoundaryChunker(); SentenceBoundaryChunkingSettings chunkingSettings = new SentenceBoundaryChunkingSettings(maxWordsPerChunk, 0); - var chunks = chunker.chunk(TEST_TEXT, chunkingSettings); + var chunks = textChunks(chunker, TEST_TEXT, maxWordsPerChunk, false); int numChunks = expectedNumberOfChunks(sentenceSizes(TEST_TEXT), maxWordsPerChunk); assertThat("words per chunk " + maxWordsPerChunk, chunks, hasSize(numChunks)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java index ef643a4b36fdc..2ef28f2cf2e77 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java @@ -14,6 +14,7 @@ import java.util.List; import java.util.Locale; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; @@ -65,9 +66,22 @@ public class WordBoundaryChunkerTests extends ESTestCase { NUM_WORDS_IN_TEST_TEXT = wordCount; } + /** + * Utility method for testing. + * Use the chunk functions that return offsets where possible + */ + List textChunks(WordBoundaryChunker chunker, String input, int chunkSize, int overlap) { + if (input.isEmpty()) { + return List.of(""); + } + + var chunkPositions = chunker.chunk(input, chunkSize, overlap); + return chunkPositions.stream().map(p -> input.substring(p.start(), p.end())).collect(Collectors.toList()); + } + public void testSingleSplit() { var chunker = new WordBoundaryChunker(); - var chunks = chunker.chunk(TEST_TEXT, 10_000, 0); + var chunks = textChunks(chunker, TEST_TEXT, 10_000, 0); assertThat(chunks, hasSize(1)); assertEquals(TEST_TEXT, chunks.get(0)); } @@ -168,11 +182,11 @@ public void testWindowSpanningWords() { } var whiteSpacedText = input.toString().stripTrailing(); - var chunks = new WordBoundaryChunker().chunk(whiteSpacedText, 20, 10); + var chunks = textChunks(new WordBoundaryChunker(), whiteSpacedText, 20, 10); assertChunkContents(chunks, numWords, 20, 10); - chunks = new WordBoundaryChunker().chunk(whiteSpacedText, 10, 4); + chunks = textChunks(new WordBoundaryChunker(), whiteSpacedText, 10, 4); assertChunkContents(chunks, numWords, 10, 4); - chunks = new WordBoundaryChunker().chunk(whiteSpacedText, 15, 3); + chunks = textChunks(new WordBoundaryChunker(), whiteSpacedText, 15, 3); assertChunkContents(chunks, numWords, 15, 3); } @@ -217,28 +231,28 @@ public void testWindowSpanning_TextShorterThanWindow() { } public void testEmptyString() { - var chunks = new WordBoundaryChunker().chunk("", 10, 5); - assertThat(chunks, contains("")); + var chunks = textChunks(new WordBoundaryChunker(), "", 10, 5); + assertThat(chunks.toString(), chunks, contains("")); } public void testWhitespace() { - var chunks = new WordBoundaryChunker().chunk(" ", 10, 5); + var chunks = textChunks(new WordBoundaryChunker(), " ", 10, 5); assertThat(chunks, contains(" ")); } public void testPunctuation() { int chunkSize = 1; - var chunks = new WordBoundaryChunker().chunk("Comma, separated", chunkSize, 0); + var chunks = textChunks(new WordBoundaryChunker(), "Comma, separated", chunkSize, 0); assertThat(chunks, contains("Comma", ", separated")); - chunks = new WordBoundaryChunker().chunk("Mme. Thénardier", chunkSize, 0); + chunks = textChunks(new WordBoundaryChunker(), "Mme. Thénardier", chunkSize, 0); assertThat(chunks, contains("Mme", ". Thénardier")); - chunks = new WordBoundaryChunker().chunk("Won't you chunk", chunkSize, 0); + chunks = textChunks(new WordBoundaryChunker(), "Won't you chunk", chunkSize, 0); assertThat(chunks, contains("Won't", " you", " chunk")); chunkSize = 10; - chunks = new WordBoundaryChunker().chunk("Won't you chunk", chunkSize, 0); + chunks = textChunks(new WordBoundaryChunker(), "Won't you chunk", chunkSize, 0); assertThat(chunks, contains("Won't you chunk")); } From 22a392f1b69224aac3894dd6a05b892ccbb6a75d Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 9 Dec 2024 07:33:11 -0800 Subject: [PATCH 131/204] Remove client.type setting (#118192) The client.type setting is a holdover from the node client which was removed in 8.0. The setting has been a noop since 8.0. This commit removes the setting. relates #104574 --- docs/changelog/118192.yaml | 11 +++++++++++ .../org/elasticsearch/client/internal/Client.java | 10 ---------- .../common/settings/ClusterSettings.java | 2 -- 3 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 docs/changelog/118192.yaml diff --git a/docs/changelog/118192.yaml b/docs/changelog/118192.yaml new file mode 100644 index 0000000000000..03542048761d3 --- /dev/null +++ b/docs/changelog/118192.yaml @@ -0,0 +1,11 @@ +pr: 118192 +summary: Remove `client.type` setting +area: Infra/Core +type: breaking +issues: [104574] +breaking: + title: Remove `client.type` setting + area: Cluster and node setting + details: The node setting `client.type` has been ignored since the node client was removed in 8.0. The setting is now removed. + impact: Remove the `client.type` setting from `elasticsearch.yml` + notable: false diff --git a/server/src/main/java/org/elasticsearch/client/internal/Client.java b/server/src/main/java/org/elasticsearch/client/internal/Client.java index 4158bbfb27cda..2d1cbe0cce7f7 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/Client.java +++ b/server/src/main/java/org/elasticsearch/client/internal/Client.java @@ -52,8 +52,6 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; import org.elasticsearch.transport.RemoteClusterService; @@ -74,14 +72,6 @@ */ public interface Client extends ElasticsearchClient { - // Note: This setting is registered only for bwc. The value is never read. - Setting CLIENT_TYPE_SETTING_S = new Setting<>("client.type", "node", (s) -> { - return switch (s) { - case "node", "transport" -> s; - default -> throw new IllegalArgumentException("Can't parse [client.type] must be one of [node, transport]"); - }; - }, Property.NodeScope, Property.Deprecated); - /** * The admin client that can be used to perform administrative operations. */ diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index a9a9411de8e1f..16af7ca2915d4 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -20,7 +20,6 @@ import org.elasticsearch.action.support.DestructiveOperations; import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.bootstrap.BootstrapSettings; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.InternalClusterInfoService; @@ -483,7 +482,6 @@ public void apply(Settings value, Settings current, Settings previous) { AutoCreateIndex.AUTO_CREATE_INDEX_SETTING, BaseRestHandler.MULTI_ALLOW_EXPLICIT_INDEX, ClusterName.CLUSTER_NAME_SETTING, - Client.CLIENT_TYPE_SETTING_S, ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING, EsExecutors.NODE_PROCESSORS_SETTING, ThreadContext.DEFAULT_HEADERS_SETTING, From 0586cbfb34c7201a996578db60d12fea8594261c Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 9 Dec 2024 15:46:22 +0000 Subject: [PATCH 132/204] Remove unused `BlobStore#deleteBlobsIgnoringIfNotExists` (#118245) This method is never called against a general `BlobStore`, we only use it in certain implementations for which a bulk delete at the `BlobStore` level makes sense. This commit removes the unused interface method. --- .../azure/AzureBlobContainer.java | 2 +- .../repositories/azure/AzureBlobStore.java | 3 +- .../azure/AzureBlobContainerStatsTests.java | 9 ++--- .../gcs/GoogleCloudStorageBlobContainer.java | 4 +-- .../gcs/GoogleCloudStorageBlobStore.java | 10 ++---- .../repositories/s3/S3BlobContainer.java | 6 ++-- .../repositories/s3/S3BlobStore.java | 3 +- .../common/blobstore/url/URLBlobStore.java | 8 ----- .../repositories/hdfs/HdfsBlobStore.java | 7 ---- .../hdfs/HdfsBlobStoreRepositoryTests.java | 5 --- ...BlobStoreRepositoryOperationPurposeIT.java | 6 ---- .../common/blobstore/BlobStore.java | 10 ------ .../common/blobstore/fs/FsBlobContainer.java | 2 +- .../common/blobstore/fs/FsBlobStore.java | 4 +-- ...bStoreRepositoryDeleteThrottlingTests.java | 6 ---- .../LatencySimulatingBlobStoreRepository.java | 6 ---- .../ESBlobStoreRepositoryIntegTestCase.java | 35 ------------------- .../snapshots/mockstore/BlobStoreWrapper.java | 9 +---- ...archableSnapshotsPrewarmingIntegTests.java | 7 ---- .../analyze/RepositoryAnalysisFailureIT.java | 5 --- .../analyze/RepositoryAnalysisSuccessIT.java | 5 --- 21 files changed, 17 insertions(+), 135 deletions(-) diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java index 73936d82fc204..08bdc2051b9e3 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java @@ -144,7 +144,7 @@ public DeleteResult delete(OperationPurpose purpose) throws IOException { @Override public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - blobStore.deleteBlobsIgnoringIfNotExists(purpose, new Iterator<>() { + blobStore.deleteBlobs(purpose, new Iterator<>() { @Override public boolean hasNext() { return blobNames.hasNext(); diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java index e4f973fb73a4e..3cac0dc4bb6db 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java @@ -264,8 +264,7 @@ public DeleteResult deleteBlobDirectory(OperationPurpose purpose, String path) t return new DeleteResult(blobsDeleted.get(), bytesDeleted.get()); } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { + void deleteBlobs(OperationPurpose purpose, Iterator blobNames) { if (blobNames.hasNext() == false) { return; } diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java index f6e97187222e7..8979507230bdd 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java @@ -72,7 +72,7 @@ public void testRetriesAndOperationsAreTrackedSeparately() throws IOException { false ); case LIST_BLOBS -> blobStore.listBlobsByPrefix(purpose, randomIdentifier(), randomIdentifier()); - case BLOB_BATCH -> blobStore.deleteBlobsIgnoringIfNotExists( + case BLOB_BATCH -> blobStore.deleteBlobs( purpose, List.of(randomIdentifier(), randomIdentifier(), randomIdentifier()).iterator() ); @@ -113,7 +113,7 @@ public void testOperationPurposeIsReflectedInBlobStoreStats() throws IOException os.flush(); }); // BLOB_BATCH - blobStore.deleteBlobsIgnoringIfNotExists(purpose, List.of(randomIdentifier(), randomIdentifier(), randomIdentifier()).iterator()); + blobStore.deleteBlobs(purpose, List.of(randomIdentifier(), randomIdentifier(), randomIdentifier()).iterator()); Map stats = blobStore.stats(); String statsMapString = stats.toString(); @@ -148,10 +148,7 @@ public void testOperationPurposeIsNotReflectedInBlobStoreStatsWhenNotServerless( os.flush(); }); // BLOB_BATCH - blobStore.deleteBlobsIgnoringIfNotExists( - purpose, - List.of(randomIdentifier(), randomIdentifier(), randomIdentifier()).iterator() - ); + blobStore.deleteBlobs(purpose, List.of(randomIdentifier(), randomIdentifier(), randomIdentifier()).iterator()); } Map stats = blobStore.stats(); diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java index 047549cc893ed..edcf03580da09 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java @@ -114,12 +114,12 @@ public void writeBlobAtomic(OperationPurpose purpose, String blobName, BytesRefe @Override public DeleteResult delete(OperationPurpose purpose) throws IOException { - return blobStore.deleteDirectory(purpose, path().buildAsString()); + return blobStore.deleteDirectory(path().buildAsString()); } @Override public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - blobStore.deleteBlobsIgnoringIfNotExists(purpose, new Iterator<>() { + blobStore.deleteBlobs(new Iterator<>() { @Override public boolean hasNext() { return blobNames.hasNext(); diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java index 9cbf64e7e0146..c68217a1a3738 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.BlobStoreActionStats; import org.elasticsearch.common.blobstore.DeleteResult; -import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.blobstore.OptionalBytesReference; import org.elasticsearch.common.blobstore.support.BlobContainerUtils; import org.elasticsearch.common.blobstore.support.BlobMetadata; @@ -491,10 +490,9 @@ private void writeBlobMultipart(BlobInfo blobInfo, byte[] buffer, int offset, in /** * Deletes the given path and all its children. * - * @param purpose The purpose of the delete operation * @param pathStr Name of path to delete */ - DeleteResult deleteDirectory(OperationPurpose purpose, String pathStr) throws IOException { + DeleteResult deleteDirectory(String pathStr) throws IOException { return SocketAccess.doPrivilegedIOException(() -> { DeleteResult deleteResult = DeleteResult.ZERO; Page page = client().list(bucketName, BlobListOption.prefix(pathStr)); @@ -502,7 +500,7 @@ DeleteResult deleteDirectory(OperationPurpose purpose, String pathStr) throws IO final AtomicLong blobsDeleted = new AtomicLong(0L); final AtomicLong bytesDeleted = new AtomicLong(0L); final Iterator blobs = page.getValues().iterator(); - deleteBlobsIgnoringIfNotExists(purpose, new Iterator<>() { + deleteBlobs(new Iterator<>() { @Override public boolean hasNext() { return blobs.hasNext(); @@ -526,11 +524,9 @@ public String next() { /** * Deletes multiple blobs from the specific bucket using a batch request * - * @param purpose the purpose of the delete operation * @param blobNames names of the blobs to delete */ - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { + void deleteBlobs(Iterator blobNames) throws IOException { if (blobNames.hasNext() == false) { return; } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java index e13cc40dd3e0f..f527dcd42814c 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java @@ -342,10 +342,10 @@ public DeleteResult delete(OperationPurpose purpose) throws IOException { return summary.getKey(); }); if (list.isTruncated()) { - blobStore.deleteBlobsIgnoringIfNotExists(purpose, blobNameIterator); + blobStore.deleteBlobs(purpose, blobNameIterator); prevListing = list; } else { - blobStore.deleteBlobsIgnoringIfNotExists(purpose, Iterators.concat(blobNameIterator, Iterators.single(keyPath))); + blobStore.deleteBlobs(purpose, Iterators.concat(blobNameIterator, Iterators.single(keyPath))); break; } } @@ -357,7 +357,7 @@ public DeleteResult delete(OperationPurpose purpose) throws IOException { @Override public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - blobStore.deleteBlobsIgnoringIfNotExists(purpose, Iterators.map(blobNames, this::buildKey)); + blobStore.deleteBlobs(purpose, Iterators.map(blobNames, this::buildKey)); } @Override diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 4f2b0f213e448..4bd54aa37077f 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -340,8 +340,7 @@ public BlobContainer blobContainer(BlobPath path) { return new S3BlobContainer(path, this); } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { + void deleteBlobs(OperationPurpose purpose, Iterator blobNames) throws IOException { if (blobNames.hasNext() == false) { return; } diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java index 8538d2ba673bc..0e9c735b22fd6 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.BlobStoreException; -import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.blobstore.url.http.HttpURLBlobContainer; import org.elasticsearch.common.blobstore.url.http.URLHttpClient; import org.elasticsearch.common.blobstore.url.http.URLHttpClientSettings; @@ -23,10 +22,8 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.CheckedFunction; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.util.Iterator; import java.util.List; /** @@ -109,11 +106,6 @@ public BlobContainer blobContainer(BlobPath blobPath) { } } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - throw new UnsupportedOperationException("Bulk deletes are not supported in URL repositories"); - } - @Override public void close() { // nothing to do here... diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java index eaf2429ae6258..e817384d95c04 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java @@ -16,10 +16,8 @@ import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.common.blobstore.OperationPurpose; import java.io.IOException; -import java.util.Iterator; final class HdfsBlobStore implements BlobStore { @@ -72,11 +70,6 @@ public BlobContainer blobContainer(BlobPath path) { return new HdfsBlobContainer(path, this, buildHdfsPath(path), bufferSize, securityContext, replicationFactor); } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - throw new UnsupportedOperationException("Bulk deletes are not supported in Hdfs repositories"); - } - private Path buildHdfsPath(BlobPath blobPath) { final Path path = translateToHdfsPath(blobPath); if (readOnly == false) { diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreRepositoryTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreRepositoryTests.java index 17927b02a08dc..3e1c112a4d9f7 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreRepositoryTests.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreRepositoryTests.java @@ -46,11 +46,6 @@ public void testSnapshotAndRestore() throws Exception { testSnapshotAndRestore(false); } - @Override - public void testBlobStoreBulkDeletion() throws Exception { - // HDFS does not implement bulk deletion from different BlobContainers - } - @Override protected Collection> nodePlugins() { return Collections.singletonList(HdfsPlugin.class); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java index c0a2c83f7fe1e..b2e02b2f4c271 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java @@ -36,7 +36,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.Collection; -import java.util.Iterator; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -136,11 +135,6 @@ public BlobContainer blobContainer(BlobPath path) { return new AssertingBlobContainer(delegateBlobStore.blobContainer(path)); } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - delegateBlobStore.deleteBlobsIgnoringIfNotExists(purpose, blobNames); - } - @Override public void close() throws IOException { delegateBlobStore.close(); diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/BlobStore.java b/server/src/main/java/org/elasticsearch/common/blobstore/BlobStore.java index d67c034fd3e27..f1fe028f60f6e 100644 --- a/server/src/main/java/org/elasticsearch/common/blobstore/BlobStore.java +++ b/server/src/main/java/org/elasticsearch/common/blobstore/BlobStore.java @@ -9,9 +9,7 @@ package org.elasticsearch.common.blobstore; import java.io.Closeable; -import java.io.IOException; import java.util.Collections; -import java.util.Iterator; import java.util.Map; /** @@ -28,14 +26,6 @@ public interface BlobStore extends Closeable { */ BlobContainer blobContainer(BlobPath path); - /** - * Delete all the provided blobs from the blob store. Each blob could belong to a different {@code BlobContainer} - * - * @param purpose the purpose of the delete operation - * @param blobNames the blobs to be deleted - */ - void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException; - /** * Returns statistics on the count of operations that have been performed on this blob store */ diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java index b5118d8a289a9..7d40008231292 100644 --- a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java +++ b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java @@ -177,7 +177,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO @Override public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - blobStore.deleteBlobsIgnoringIfNotExists(purpose, Iterators.map(blobNames, blobName -> path.resolve(blobName).toString())); + blobStore.deleteBlobs(Iterators.map(blobNames, blobName -> path.resolve(blobName).toString())); } @Override diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java index 53e3b4b4796dc..9a368483d46c0 100644 --- a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java +++ b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.core.IOUtils; import java.io.IOException; @@ -70,8 +69,7 @@ public BlobContainer blobContainer(BlobPath path) { return new FsBlobContainer(this, path, f); } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { + void deleteBlobs(Iterator blobNames) throws IOException { IOException ioe = null; long suppressedExceptions = 0; while (blobNames.hasNext()) { diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryDeleteThrottlingTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryDeleteThrottlingTests.java index 0b5999b614050..4facaa391ec24 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryDeleteThrottlingTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryDeleteThrottlingTests.java @@ -35,7 +35,6 @@ import java.io.OutputStream; import java.util.Collection; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -100,11 +99,6 @@ public BlobContainer blobContainer(BlobPath path) { return new ConcurrencyLimitingBlobContainer(delegate.blobContainer(path), activeIndices, countDownLatch); } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - delegate.deleteBlobsIgnoringIfNotExists(purpose, blobNames); - } - @Override public void close() throws IOException { delegate.close(); diff --git a/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepository.java b/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepository.java index f360a6c012cb7..cd2812a95cfac 100644 --- a/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepository.java +++ b/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepository.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.io.InputStream; -import java.util.Iterator; class LatencySimulatingBlobStoreRepository extends FsRepository { @@ -53,11 +52,6 @@ public BlobContainer blobContainer(BlobPath path) { return new LatencySimulatingBlobContainer(blobContainer); } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - fsBlobStore.deleteBlobsIgnoringIfNotExists(purpose, blobNames); - } - @Override public void close() throws IOException { fsBlobStore.close(); diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java index b85ee970664e2..c982f36e5ccb3 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java @@ -48,7 +48,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.NoSuchFileException; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -70,7 +69,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -524,39 +522,6 @@ public void testIndicesDeletedFromRepository() throws Exception { assertAcked(clusterAdmin().prepareDeleteSnapshot(TEST_REQUEST_TIMEOUT, repoName, "test-snap2").get()); } - public void testBlobStoreBulkDeletion() throws Exception { - Map> expectedBlobsPerContainer = new HashMap<>(); - try (BlobStore store = newBlobStore()) { - List blobsToDelete = new ArrayList<>(); - int numberOfContainers = randomIntBetween(2, 5); - for (int i = 0; i < numberOfContainers; i++) { - BlobPath containerPath = BlobPath.EMPTY.add(randomIdentifier()); - final BlobContainer container = store.blobContainer(containerPath); - int numberOfBlobsPerContainer = randomIntBetween(5, 10); - for (int j = 0; j < numberOfBlobsPerContainer; j++) { - byte[] bytes = randomBytes(randomInt(100)); - String blobName = randomAlphaOfLength(10); - container.writeBlob(randomPurpose(), blobName, new BytesArray(bytes), false); - if (randomBoolean()) { - blobsToDelete.add(containerPath.buildAsString() + blobName); - } else { - expectedBlobsPerContainer.computeIfAbsent(containerPath, unused -> new ArrayList<>()).add(blobName); - } - } - } - - store.deleteBlobsIgnoringIfNotExists(randomPurpose(), blobsToDelete.iterator()); - for (var containerEntry : expectedBlobsPerContainer.entrySet()) { - BlobContainer blobContainer = store.blobContainer(containerEntry.getKey()); - Map blobsInContainer = blobContainer.listBlobs(randomPurpose()); - for (String expectedBlob : containerEntry.getValue()) { - assertThat(blobsInContainer, hasKey(expectedBlob)); - } - blobContainer.delete(randomPurpose()); - } - } - } - public void testDanglingShardLevelBlobCleanup() throws Exception { final var repoName = createRepository(randomRepositoryName()); final var client = client(); diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/BlobStoreWrapper.java b/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/BlobStoreWrapper.java index 5803c2a825671..54af75fc584d6 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/BlobStoreWrapper.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/BlobStoreWrapper.java @@ -12,15 +12,13 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.BlobStoreActionStats; -import org.elasticsearch.common.blobstore.OperationPurpose; import java.io.IOException; -import java.util.Iterator; import java.util.Map; public class BlobStoreWrapper implements BlobStore { - private BlobStore delegate; + private final BlobStore delegate; public BlobStoreWrapper(BlobStore delegate) { this.delegate = delegate; @@ -31,11 +29,6 @@ public BlobContainer blobContainer(BlobPath path) { return delegate.blobContainer(path); } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - delegate.deleteBlobsIgnoringIfNotExists(purpose, blobNames); - } - @Override public void close() throws IOException { delegate.close(); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java index ab38a89870500..c955457b78d60 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java @@ -67,7 +67,6 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -466,12 +465,6 @@ public BlobContainer blobContainer(BlobPath path) { return new TrackingFilesBlobContainer(delegate.blobContainer(path)); } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) - throws IOException { - delegate.deleteBlobsIgnoringIfNotExists(purpose, blobNames); - } - @Override public void close() throws IOException { delegate.close(); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java index b8acd9808a35e..6a638f53a6330 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java @@ -569,11 +569,6 @@ public BlobContainer blobContainer(BlobPath path) { } } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) { - assertPurpose(purpose); - } - private void deleteContainer(DisruptableBlobContainer container) { blobContainer = null; } diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java index 1f8b247e76176..c24a254d34ace 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java @@ -287,11 +287,6 @@ private void deleteContainer(AssertingBlobContainer container) { } } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) { - assertPurpose(purpose); - } - @Override public void close() {} From 5e859d9301ffe736548dfc2b6e72807a7f9006ff Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Mon, 9 Dec 2024 11:06:27 -0500 Subject: [PATCH 133/204] Even better(er) binary quantization (#117994) This measurably improves BBQ by adjusting the underlying algorithm to an optimized per vector scalar quantization. This is a brand new way to quantize vectors. Instead of there being a global set of upper and lower quantile bands, these are optimized and calculated per individual vector. Additionally, vectors are centered on a common centroid. This allows for an almost 32x reduction in memory, and even better recall than before at the cost of slightly increasing indexing time. Additionally, this new approach is easily generalizable to various other bit sizes (e.g. 2 bits, etc.). While not taken advantage of yet, we may update our scalar quantized indices in the future to use this new algorithm, giving significant boosts in recall. The recall gains spread from 2% to almost 10% for certain datasets with an additional 5-10% indexing cost when indexing with HNSW when compared with current BBQ. --- docs/changelog/117994.yaml | 5 + rest-api-spec/build.gradle | 2 + .../search.vectors/41_knn_search_bbq_hnsw.yml | 66 +- .../search.vectors/42_knn_search_bbq_flat.yml | 66 +- server/src/main/java/module-info.java | 4 +- .../index/codec/vectors/BQVectorUtils.java | 14 + .../es816/ES816BinaryFlatVectorsScorer.java | 59 +- .../ES816BinaryQuantizedVectorsFormat.java | 2 +- ...ES816HnswBinaryQuantizedVectorsFormat.java | 17 +- .../es818/BinarizedByteVectorValues.java | 87 ++ .../es818/ES818BinaryFlatVectorsScorer.java | 188 ++++ .../ES818BinaryQuantizedVectorsFormat.java | 132 +++ .../ES818BinaryQuantizedVectorsReader.java | 412 ++++++++ .../ES818BinaryQuantizedVectorsWriter.java | 944 ++++++++++++++++++ ...ES818HnswBinaryQuantizedVectorsFormat.java | 145 +++ .../es818/OffHeapBinarizedVectorValues.java | 371 +++++++ .../es818/OptimizedScalarQuantizer.java | 246 +++++ .../vectors/DenseVectorFieldMapper.java | 8 +- .../action/search/SearchCapabilities.java | 2 + .../org.apache.lucene.codecs.KnnVectorsFormat | 2 + .../codec/vectors/BQVectorUtilsTests.java | 26 + .../es816/ES816BinaryFlatRWVectorsScorer.java | 256 +++++ .../ES816BinaryFlatVectorsScorerTests.java | 12 +- .../ES816BinaryQuantizedRWVectorsFormat.java | 52 + ...S816BinaryQuantizedVectorsFormatTests.java | 2 +- .../ES816BinaryQuantizedVectorsWriter.java | 4 +- ...816HnswBinaryQuantizedRWVectorsFormat.java | 55 + ...HnswBinaryQuantizedVectorsFormatTests.java | 2 +- ...S818BinaryQuantizedVectorsFormatTests.java | 181 ++++ ...HnswBinaryQuantizedVectorsFormatTests.java | 132 +++ .../es818/OptimizedScalarQuantizerTests.java | 136 +++ .../vectors/DenseVectorFieldMapperTests.java | 8 +- 32 files changed, 3501 insertions(+), 137 deletions(-) create mode 100644 docs/changelog/117994.yaml create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es818/BinarizedByteVectorValues.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryFlatVectorsScorer.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormat.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsWriter.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormat.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es818/OffHeapBinarizedVectorValues.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es818/OptimizedScalarQuantizer.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatRWVectorsScorer.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedRWVectorsFormat.java rename server/src/{main => test}/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java (99%) create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedRWVectorsFormat.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/es818/OptimizedScalarQuantizerTests.java diff --git a/docs/changelog/117994.yaml b/docs/changelog/117994.yaml new file mode 100644 index 0000000000000..603f2d855a11a --- /dev/null +++ b/docs/changelog/117994.yaml @@ -0,0 +1,5 @@ +pr: 117994 +summary: Even better(er) binary quantization +area: Vector Search +type: enhancement +issues: [] diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index e2af894eb0939..7347d9c1312dd 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -67,4 +67,6 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("logsdb/20_source_mapping/include/exclude is supported with stored _source", "no longer serialize source_mode") task.skipTest("logsdb/20_source_mapping/synthetic _source is default", "no longer serialize source_mode") task.skipTest("search/520_fetch_fields/fetch _seq_no via fields", "error code is changed from 5xx to 400 in 9.0") + task.skipTest("search.vectors/41_knn_search_bbq_hnsw/Test knn search", "Scoring has changed in latest versions") + task.skipTest("search.vectors/42_knn_search_bbq_flat/Test knn search", "Scoring has changed in latest versions") }) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml index 188c155e4a836..5767c895fbe7e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml @@ -11,20 +11,11 @@ setup: number_of_shards: 1 mappings: properties: - name: - type: keyword vector: type: dense_vector dims: 64 index: true - similarity: l2_norm - index_options: - type: bbq_hnsw - another_vector: - type: dense_vector - dims: 64 - index: true - similarity: l2_norm + similarity: max_inner_product index_options: type: bbq_hnsw @@ -33,9 +24,14 @@ setup: index: bbq_hnsw id: "1" body: - name: cow.jpg - vector: [300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0] - another_vector: [115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0] + vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] # Flush in order to provoke a merge later - do: indices.flush: @@ -46,9 +42,14 @@ setup: index: bbq_hnsw id: "2" body: - name: moose.jpg - vector: [100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0] - another_vector: [50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120] + vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] # Flush in order to provoke a merge later - do: indices.flush: @@ -60,8 +61,14 @@ setup: id: "3" body: name: rabbit.jpg - vector: [111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0] - another_vector: [11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0] + vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.657, 1.285, 0.2 , -0.062, 0.038, 0.089, -0.068, -0.058] # Flush in order to provoke a merge later - do: indices.flush: @@ -73,20 +80,33 @@ setup: max_num_segments: 1 --- "Test knn search": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ optimized_scalar_quantization_bbq ] + test_runner_features: capabilities + reason: "BBQ scoring improved and changed with optimized_scalar_quantization_bbq" - do: search: index: bbq_hnsw body: knn: field: vector - query_vector: [ 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0] + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] k: 3 num_candidates: 3 - # Depending on how things are distributed, docs 2 and 3 might be swapped - # here we verify that are last hit is always the worst one - - match: { hits.hits.2._id: "1" } - + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "2" } --- "Test bad quantization parameters": - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml index ed7a8dd5df65d..dcdae04aeabb4 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml @@ -11,20 +11,11 @@ setup: number_of_shards: 1 mappings: properties: - name: - type: keyword vector: type: dense_vector dims: 64 index: true - similarity: l2_norm - index_options: - type: bbq_flat - another_vector: - type: dense_vector - dims: 64 - index: true - similarity: l2_norm + similarity: max_inner_product index_options: type: bbq_flat @@ -33,9 +24,14 @@ setup: index: bbq_flat id: "1" body: - name: cow.jpg - vector: [300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0, 230.0, 300.33, -34.8988, 15.555, -200.0] - another_vector: [115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0, 130.0, 115.0, -1.02, 15.555, -100.0] + vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] # Flush in order to provoke a merge later - do: indices.flush: @@ -46,9 +42,14 @@ setup: index: bbq_flat id: "2" body: - name: moose.jpg - vector: [100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0, -0.5, 100.0, -13, 14.8, -156.0] - another_vector: [50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120, -0.5, 50.0, -1, 1, 120] + vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] # Flush in order to provoke a merge later - do: indices.flush: @@ -59,9 +60,14 @@ setup: index: bbq_flat id: "3" body: - name: rabbit.jpg - vector: [111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0, 0.5, 111.3, -13.0, 14.8, -156.0] - another_vector: [11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0, -0.5, 11.0, 0, 12, 111.0] + vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.657, 1.285, 0.2 , -0.062, 0.038, 0.089, -0.068, -0.058] # Flush in order to provoke a merge later - do: indices.flush: @@ -73,19 +79,33 @@ setup: max_num_segments: 1 --- "Test knn search": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ optimized_scalar_quantization_bbq ] + test_runner_features: capabilities + reason: "BBQ scoring improved and changed with optimized_scalar_quantization_bbq" - do: search: index: bbq_flat body: knn: field: vector - query_vector: [ 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0, -0.5, 90.0, -10, 14.8, -156.0] + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] k: 3 num_candidates: 3 - # Depending on how things are distributed, docs 2 and 3 might be swapped - # here we verify that are last hit is always the worst one - - match: { hits.hits.2._id: "1" } + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "2" } --- "Test bad parameters": - do: diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 331a2bc0dddac..ff902dbede007 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -459,7 +459,9 @@ org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat, org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat, org.elasticsearch.index.codec.vectors.es816.ES816BinaryQuantizedVectorsFormat, - org.elasticsearch.index.codec.vectors.es816.ES816HnswBinaryQuantizedVectorsFormat; + org.elasticsearch.index.codec.vectors.es816.ES816HnswBinaryQuantizedVectorsFormat, + org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat, + org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat; provides org.apache.lucene.codecs.Codec with diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java index 5201e57179cc7..1aff06a175967 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java @@ -40,6 +40,20 @@ public static boolean isUnitVector(float[] v) { return Math.abs(l1norm - 1.0d) <= EPSILON; } + public static void packAsBinary(byte[] vector, byte[] packed) { + for (int i = 0; i < vector.length;) { + byte result = 0; + for (int j = 7; j >= 0 && i < vector.length; j--) { + assert vector[i] == 0 || vector[i] == 1; + result |= (byte) ((vector[i] & 1) << j); + ++i; + } + int index = ((i + 7) / 8) - 1; + assert index < packed.length; + packed[index] = result; + } + } + public static int discretize(int value, int bucket) { return ((value + (bucket - 1)) / bucket) * bucket; } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatVectorsScorer.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatVectorsScorer.java index 445bdadab2354..e85079e998c61 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatVectorsScorer.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatVectorsScorer.java @@ -48,13 +48,8 @@ class ES816BinaryFlatVectorsScorer implements FlatVectorsScorer { public RandomVectorScorerSupplier getRandomVectorScorerSupplier( VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues - ) throws IOException { - if (vectorValues instanceof BinarizedByteVectorValues) { - throw new UnsupportedOperationException( - "getRandomVectorScorerSupplier(VectorSimilarityFunction,RandomAccessVectorValues) not implemented for binarized format" - ); - } - return nonQuantizedDelegate.getRandomVectorScorerSupplier(similarityFunction, vectorValues); + ) { + throw new UnsupportedOperationException(); } @Override @@ -90,61 +85,11 @@ public RandomVectorScorer getRandomVectorScorer( return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target); } - RandomVectorScorerSupplier getRandomVectorScorerSupplier( - VectorSimilarityFunction similarityFunction, - ES816BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues scoringVectors, - BinarizedByteVectorValues targetVectors - ) { - return new BinarizedRandomVectorScorerSupplier(scoringVectors, targetVectors, similarityFunction); - } - @Override public String toString() { return "ES816BinaryFlatVectorsScorer(nonQuantizedDelegate=" + nonQuantizedDelegate + ")"; } - /** Vector scorer supplier over binarized vector values */ - static class BinarizedRandomVectorScorerSupplier implements RandomVectorScorerSupplier { - private final ES816BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues queryVectors; - private final BinarizedByteVectorValues targetVectors; - private final VectorSimilarityFunction similarityFunction; - - BinarizedRandomVectorScorerSupplier( - ES816BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues queryVectors, - BinarizedByteVectorValues targetVectors, - VectorSimilarityFunction similarityFunction - ) { - this.queryVectors = queryVectors; - this.targetVectors = targetVectors; - this.similarityFunction = similarityFunction; - } - - @Override - public RandomVectorScorer scorer(int ord) throws IOException { - byte[] vector = queryVectors.vectorValue(ord); - int quantizedSum = queryVectors.sumQuantizedValues(ord); - float distanceToCentroid = queryVectors.getCentroidDistance(ord); - float lower = queryVectors.getLower(ord); - float width = queryVectors.getWidth(ord); - float normVmC = 0f; - float vDotC = 0f; - if (similarityFunction != EUCLIDEAN) { - normVmC = queryVectors.getNormVmC(ord); - vDotC = queryVectors.getVDotC(ord); - } - BinaryQueryVector binaryQueryVector = new BinaryQueryVector( - vector, - new BinaryQuantizer.QueryFactors(quantizedSum, distanceToCentroid, lower, width, normVmC, vDotC) - ); - return new BinarizedRandomVectorScorer(binaryQueryVector, targetVectors, similarityFunction); - } - - @Override - public RandomVectorScorerSupplier copy() throws IOException { - return new BinarizedRandomVectorScorerSupplier(queryVectors.copy(), targetVectors.copy(), similarityFunction); - } - } - /** A binarized query representing its quantized form along with factors */ record BinaryQueryVector(byte[] vector, BinaryQuantizer.QueryFactors factors) {} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormat.java index d864ec5dee8c5..61b6edc474d1f 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormat.java @@ -62,7 +62,7 @@ public ES816BinaryQuantizedVectorsFormat() { @Override public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - return new ES816BinaryQuantizedVectorsWriter(scorer, rawVectorFormat.fieldsWriter(state), state); + throw new UnsupportedOperationException(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormat.java index 52f9f14b7bf97..1dbb4e432b188 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormat.java @@ -25,10 +25,8 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.search.TaskExecutor; import org.apache.lucene.util.hnsw.HnswGraph; import java.io.IOException; @@ -52,21 +50,18 @@ public class ES816HnswBinaryQuantizedVectorsFormat extends KnnVectorsFormat { * Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to * {@link Lucene99HnswVectorsFormat#DEFAULT_MAX_CONN}. See {@link HnswGraph} for more details. */ - private final int maxConn; + protected final int maxConn; /** * The number of candidate neighbors to track while searching the graph for each newly inserted * node. Defaults to {@link Lucene99HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link HnswGraph} * for details. */ - private final int beamWidth; + protected final int beamWidth; /** The format for storing, reading, merging vectors on disk */ private static final FlatVectorsFormat flatVectorsFormat = new ES816BinaryQuantizedVectorsFormat(); - private final int numMergeWorkers; - private final TaskExecutor mergeExec; - /** Constructs a format using default graph construction parameters */ public ES816HnswBinaryQuantizedVectorsFormat() { this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, null); @@ -109,17 +104,11 @@ public ES816HnswBinaryQuantizedVectorsFormat(int maxConn, int beamWidth, int num if (numMergeWorkers == 1 && mergeExec != null) { throw new IllegalArgumentException("No executor service is needed as we'll use single thread to merge"); } - this.numMergeWorkers = numMergeWorkers; - if (mergeExec != null) { - this.mergeExec = new TaskExecutor(mergeExec); - } else { - this.mergeExec = null; - } } @Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - return new Lucene99HnswVectorsWriter(state, maxConn, beamWidth, flatVectorsFormat.fieldsWriter(state), numMergeWorkers, mergeExec); + throw new UnsupportedOperationException(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/BinarizedByteVectorValues.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/BinarizedByteVectorValues.java new file mode 100644 index 0000000000000..cc1f7b85e0f78 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/BinarizedByteVectorValues.java @@ -0,0 +1,87 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors.es818; + +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.search.VectorScorer; +import org.apache.lucene.util.VectorUtil; +import org.elasticsearch.index.codec.vectors.BQVectorUtils; + +import java.io.IOException; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + */ +abstract class BinarizedByteVectorValues extends ByteVectorValues { + + /** + * Retrieve the corrective terms for the given vector ordinal. For the dot-product family of + * distances, the corrective terms are, in order + * + *

    + *
  • the lower optimized interval + *
  • the upper optimized interval + *
  • the dot-product of the non-centered vector with the centroid + *
  • the sum of quantized components + *
+ * + * For euclidean: + * + *
    + *
  • the lower optimized interval + *
  • the upper optimized interval + *
  • the l2norm of the centered vector + *
  • the sum of quantized components + *
+ * + * @param vectorOrd the vector ordinal + * @return the corrective terms + * @throws IOException if an I/O error occurs + */ + public abstract OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int vectorOrd) throws IOException; + + /** + * @return the quantizer used to quantize the vectors + */ + public abstract OptimizedScalarQuantizer getQuantizer(); + + public abstract float[] getCentroid() throws IOException; + + int discretizedDimensions() { + return BQVectorUtils.discretize(dimension(), 64); + } + + /** + * Return a {@link VectorScorer} for the given query vector. + * + * @param query the query vector + * @return a {@link VectorScorer} instance or null + */ + public abstract VectorScorer scorer(float[] query) throws IOException; + + @Override + public abstract BinarizedByteVectorValues copy() throws IOException; + + float getCentroidDP() throws IOException { + // this only gets executed on-merge + float[] centroid = getCentroid(); + return VectorUtil.dotProduct(centroid, centroid); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryFlatVectorsScorer.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryFlatVectorsScorer.java new file mode 100644 index 0000000000000..7c7e470909eb3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryFlatVectorsScorer.java @@ -0,0 +1,188 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors.es818; + +import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.VectorUtil; +import org.apache.lucene.util.hnsw.RandomVectorScorer; +import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier; +import org.elasticsearch.index.codec.vectors.BQSpaceUtils; +import org.elasticsearch.simdvec.ESVectorUtil; + +import java.io.IOException; + +import static org.apache.lucene.index.VectorSimilarityFunction.COSINE; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; +import static org.apache.lucene.index.VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT; + +/** Vector scorer over binarized vector values */ +public class ES818BinaryFlatVectorsScorer implements FlatVectorsScorer { + private final FlatVectorsScorer nonQuantizedDelegate; + private static final float FOUR_BIT_SCALE = 1f / ((1 << 4) - 1); + + public ES818BinaryFlatVectorsScorer(FlatVectorsScorer nonQuantizedDelegate) { + this.nonQuantizedDelegate = nonQuantizedDelegate; + } + + @Override + public RandomVectorScorerSupplier getRandomVectorScorerSupplier( + VectorSimilarityFunction similarityFunction, + KnnVectorValues vectorValues + ) throws IOException { + if (vectorValues instanceof BinarizedByteVectorValues) { + throw new UnsupportedOperationException( + "getRandomVectorScorerSupplier(VectorSimilarityFunction,RandomAccessVectorValues) not implemented for binarized format" + ); + } + return nonQuantizedDelegate.getRandomVectorScorerSupplier(similarityFunction, vectorValues); + } + + @Override + public RandomVectorScorer getRandomVectorScorer( + VectorSimilarityFunction similarityFunction, + KnnVectorValues vectorValues, + float[] target + ) throws IOException { + if (vectorValues instanceof BinarizedByteVectorValues binarizedVectors) { + OptimizedScalarQuantizer quantizer = binarizedVectors.getQuantizer(); + float[] centroid = binarizedVectors.getCentroid(); + // We make a copy as the quantization process mutates the input + float[] copy = ArrayUtil.copyOfSubArray(target, 0, target.length); + if (similarityFunction == COSINE) { + VectorUtil.l2normalize(copy); + } + target = copy; + byte[] initial = new byte[target.length]; + byte[] quantized = new byte[BQSpaceUtils.B_QUERY * binarizedVectors.discretizedDimensions() / 8]; + OptimizedScalarQuantizer.QuantizationResult queryCorrections = quantizer.scalarQuantize(target, initial, (byte) 4, centroid); + BQSpaceUtils.transposeHalfByte(initial, quantized); + BinaryQueryVector queryVector = new BinaryQueryVector(quantized, queryCorrections); + return new BinarizedRandomVectorScorer(queryVector, binarizedVectors, similarityFunction); + } + return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target); + } + + @Override + public RandomVectorScorer getRandomVectorScorer( + VectorSimilarityFunction similarityFunction, + KnnVectorValues vectorValues, + byte[] target + ) throws IOException { + return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target); + } + + RandomVectorScorerSupplier getRandomVectorScorerSupplier( + VectorSimilarityFunction similarityFunction, + ES818BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues scoringVectors, + BinarizedByteVectorValues targetVectors + ) { + return new BinarizedRandomVectorScorerSupplier(scoringVectors, targetVectors, similarityFunction); + } + + @Override + public String toString() { + return "ES818BinaryFlatVectorsScorer(nonQuantizedDelegate=" + nonQuantizedDelegate + ")"; + } + + /** Vector scorer supplier over binarized vector values */ + static class BinarizedRandomVectorScorerSupplier implements RandomVectorScorerSupplier { + private final ES818BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues queryVectors; + private final BinarizedByteVectorValues targetVectors; + private final VectorSimilarityFunction similarityFunction; + + BinarizedRandomVectorScorerSupplier( + ES818BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues queryVectors, + BinarizedByteVectorValues targetVectors, + VectorSimilarityFunction similarityFunction + ) { + this.queryVectors = queryVectors; + this.targetVectors = targetVectors; + this.similarityFunction = similarityFunction; + } + + @Override + public RandomVectorScorer scorer(int ord) throws IOException { + byte[] vector = queryVectors.vectorValue(ord); + OptimizedScalarQuantizer.QuantizationResult correctiveTerms = queryVectors.getCorrectiveTerms(ord); + BinaryQueryVector binaryQueryVector = new BinaryQueryVector(vector, correctiveTerms); + return new BinarizedRandomVectorScorer(binaryQueryVector, targetVectors, similarityFunction); + } + + @Override + public RandomVectorScorerSupplier copy() throws IOException { + return new BinarizedRandomVectorScorerSupplier(queryVectors.copy(), targetVectors.copy(), similarityFunction); + } + } + + /** A binarized query representing its quantized form along with factors */ + public record BinaryQueryVector(byte[] vector, OptimizedScalarQuantizer.QuantizationResult quantizationResult) {} + + /** Vector scorer over binarized vector values */ + public static class BinarizedRandomVectorScorer extends RandomVectorScorer.AbstractRandomVectorScorer { + private final BinaryQueryVector queryVector; + private final BinarizedByteVectorValues targetVectors; + private final VectorSimilarityFunction similarityFunction; + + public BinarizedRandomVectorScorer( + BinaryQueryVector queryVectors, + BinarizedByteVectorValues targetVectors, + VectorSimilarityFunction similarityFunction + ) { + super(targetVectors); + this.queryVector = queryVectors; + this.targetVectors = targetVectors; + this.similarityFunction = similarityFunction; + } + + @Override + public float score(int targetOrd) throws IOException { + byte[] quantizedQuery = queryVector.vector(); + byte[] binaryCode = targetVectors.vectorValue(targetOrd); + float qcDist = ESVectorUtil.ipByteBinByte(quantizedQuery, binaryCode); + OptimizedScalarQuantizer.QuantizationResult queryCorrections = queryVector.quantizationResult(); + OptimizedScalarQuantizer.QuantizationResult indexCorrections = targetVectors.getCorrectiveTerms(targetOrd); + float x1 = indexCorrections.quantizedComponentSum(); + float ax = indexCorrections.lowerInterval(); + // Here we assume `lx` is simply bit vectors, so the scaling isn't necessary + float lx = indexCorrections.upperInterval() - ax; + float ay = queryCorrections.lowerInterval(); + float ly = (queryCorrections.upperInterval() - ay) * FOUR_BIT_SCALE; + float y1 = queryCorrections.quantizedComponentSum(); + float score = ax * ay * targetVectors.dimension() + ay * lx * x1 + ax * ly * y1 + lx * ly * qcDist; + // For euclidean, we need to invert the score and apply the additional correction, which is + // assumed to be the squared l2norm of the centroid centered vectors. + if (similarityFunction == EUCLIDEAN) { + score = queryCorrections.additionalCorrection() + indexCorrections.additionalCorrection() - 2 * score; + return Math.max(1 / (1f + score), 0); + } else { + // For cosine and max inner product, we need to apply the additional correction, which is + // assumed to be the non-centered dot-product between the vector and the centroid + score += queryCorrections.additionalCorrection() + indexCorrections.additionalCorrection() - targetVectors.getCentroidDP(); + if (similarityFunction == MAXIMUM_INNER_PRODUCT) { + return VectorUtil.scaleMaxInnerProductScore(score); + } + return Math.max((1f + score) / 2f, 0); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormat.java new file mode 100644 index 0000000000000..1dee9599f985f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormat.java @@ -0,0 +1,132 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors.es818; + +import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; + +import java.io.IOException; + +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + * Codec for encoding/decoding binary quantized vectors The binary quantization format used here + * is a per-vector optimized scalar quantization. Also see {@link + * org.elasticsearch.index.codec.vectors.es818.OptimizedScalarQuantizer}. Some of key features are: + * + *
    + *
  • Estimating the distance between two vectors using their centroid normalized distance. This + * requires some additional corrective factors, but allows for centroid normalization to occur. + *
  • Optimized scalar quantization to bit level of centroid normalized vectors. + *
  • Asymmetric quantization of vectors, where query vectors are quantized to half-byte + * precision (normalized to the centroid) and then compared directly against the single bit + * quantized vectors in the index. + *
  • Transforming the half-byte quantized query vectors in such a way that the comparison with + * single bit vectors can be done with bit arithmetic. + *
+ * + * The format is stored in two files: + * + *

.veb (vector data) file

+ * + *

Stores the binary quantized vectors in a flat format. Additionally, it stores each vector's + * corrective factors. At the end of the file, additional information is stored for vector ordinal + * to centroid ordinal mapping and sparse vector information. + * + *

    + *
  • For each vector: + *
      + *
    • [byte] the binary quantized values, each byte holds 8 bits. + *
    • [float] the optimized quantiles and an additional similarity dependent corrective factor. + *
    • short the sum of the quantized components
      • + *
    + *
  • After the vectors, sparse vector information keeping track of monotonic blocks. + *
+ * + *

.vemb (vector metadata) file

+ * + *

Stores the metadata for the vectors. This includes the number of vectors, the number of + * dimensions, and file offset information. + * + *

    + *
  • int the field number + *
  • int the vector encoding ordinal + *
  • int the vector similarity ordinal + *
  • vint the vector dimensions + *
  • vlong the offset to the vector data in the .veb file + *
  • vlong the length of the vector data in the .veb file + *
  • vint the number of vectors + *
  • [float] the centroid
    • + *
  • float the centroid square magnitude
    • + *
  • The sparse vector information, if required, mapping vector ordinal to doc ID + *
+ */ +public class ES818BinaryQuantizedVectorsFormat extends FlatVectorsFormat { + + public static final String BINARIZED_VECTOR_COMPONENT = "BVEC"; + public static final String NAME = "ES818BinaryQuantizedVectorsFormat"; + + static final int VERSION_START = 0; + static final int VERSION_CURRENT = VERSION_START; + static final String META_CODEC_NAME = "ES818BinaryQuantizedVectorsFormatMeta"; + static final String VECTOR_DATA_CODEC_NAME = "ES818BinaryQuantizedVectorsFormatData"; + static final String META_EXTENSION = "vemb"; + static final String VECTOR_DATA_EXTENSION = "veb"; + static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16; + + private static final FlatVectorsFormat rawVectorFormat = new Lucene99FlatVectorsFormat( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() + ); + + private static final ES818BinaryFlatVectorsScorer scorer = new ES818BinaryFlatVectorsScorer( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() + ); + + /** Creates a new instance with the default number of vectors per cluster. */ + public ES818BinaryQuantizedVectorsFormat() { + super(NAME); + } + + @Override + public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new ES818BinaryQuantizedVectorsWriter(scorer, rawVectorFormat.fieldsWriter(state), state); + } + + @Override + public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new ES818BinaryQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), scorer); + } + + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } + + @Override + public String toString() { + return "ES818BinaryQuantizedVectorsFormat(name=" + NAME + ", flatVectorScorer=" + scorer + ")"; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java new file mode 100644 index 0000000000000..8036b8314cdc1 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java @@ -0,0 +1,412 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors.es818; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.search.VectorScorer; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.ReadAdvice; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.SuppressForbidden; +import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; +import org.apache.lucene.util.hnsw.RandomVectorScorer; +import org.elasticsearch.index.codec.vectors.BQVectorUtils; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + */ +@SuppressForbidden(reason = "Lucene classes") +class ES818BinaryQuantizedVectorsReader extends FlatVectorsReader { + + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(ES818BinaryQuantizedVectorsReader.class); + + private final Map fields = new HashMap<>(); + private final IndexInput quantizedVectorData; + private final FlatVectorsReader rawVectorsReader; + private final ES818BinaryFlatVectorsScorer vectorScorer; + + ES818BinaryQuantizedVectorsReader( + SegmentReadState state, + FlatVectorsReader rawVectorsReader, + ES818BinaryFlatVectorsScorer vectorsScorer + ) throws IOException { + super(vectorsScorer); + this.vectorScorer = vectorsScorer; + this.rawVectorsReader = rawVectorsReader; + int versionMeta = -1; + String metaFileName = IndexFileNames.segmentFileName( + state.segmentInfo.name, + state.segmentSuffix, + org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat.META_EXTENSION + ); + boolean success = false; + try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) { + Throwable priorE = null; + try { + versionMeta = CodecUtil.checkIndexHeader( + meta, + ES818BinaryQuantizedVectorsFormat.META_CODEC_NAME, + ES818BinaryQuantizedVectorsFormat.VERSION_START, + ES818BinaryQuantizedVectorsFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + readFields(meta, state.fieldInfos); + } catch (Throwable exception) { + priorE = exception; + } finally { + CodecUtil.checkFooter(meta, priorE); + } + quantizedVectorData = openDataInput( + state, + versionMeta, + ES818BinaryQuantizedVectorsFormat.VECTOR_DATA_EXTENSION, + ES818BinaryQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME, + // Quantized vectors are accessed randomly from their node ID stored in the HNSW + // graph. + state.context.withReadAdvice(ReadAdvice.RANDOM) + ); + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + private void readFields(ChecksumIndexInput meta, FieldInfos infos) throws IOException { + for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) { + FieldInfo info = infos.fieldInfo(fieldNumber); + if (info == null) { + throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); + } + FieldEntry fieldEntry = readField(meta, info); + validateFieldEntry(info, fieldEntry); + fields.put(info.name, fieldEntry); + } + } + + static void validateFieldEntry(FieldInfo info, FieldEntry fieldEntry) { + int dimension = info.getVectorDimension(); + if (dimension != fieldEntry.dimension) { + throw new IllegalStateException( + "Inconsistent vector dimension for field=\"" + info.name + "\"; " + dimension + " != " + fieldEntry.dimension + ); + } + + int binaryDims = BQVectorUtils.discretize(dimension, 64) / 8; + long numQuantizedVectorBytes = Math.multiplyExact((binaryDims + (Float.BYTES * 3) + Short.BYTES), (long) fieldEntry.size); + if (numQuantizedVectorBytes != fieldEntry.vectorDataLength) { + throw new IllegalStateException( + "Binarized vector data length " + + fieldEntry.vectorDataLength + + " not matching size = " + + fieldEntry.size + + " * (binaryBytes=" + + binaryDims + + " + 14" + + ") = " + + numQuantizedVectorBytes + ); + } + } + + @Override + public RandomVectorScorer getRandomVectorScorer(String field, float[] target) throws IOException { + FieldEntry fi = fields.get(field); + if (fi == null) { + return null; + } + return vectorScorer.getRandomVectorScorer( + fi.similarityFunction, + OffHeapBinarizedVectorValues.load( + fi.ordToDocDISIReaderConfiguration, + fi.dimension, + fi.size, + new OptimizedScalarQuantizer(fi.similarityFunction), + fi.similarityFunction, + vectorScorer, + fi.centroid, + fi.centroidDP, + fi.vectorDataOffset, + fi.vectorDataLength, + quantizedVectorData + ), + target + ); + } + + @Override + public RandomVectorScorer getRandomVectorScorer(String field, byte[] target) throws IOException { + return rawVectorsReader.getRandomVectorScorer(field, target); + } + + @Override + public void checkIntegrity() throws IOException { + rawVectorsReader.checkIntegrity(); + CodecUtil.checksumEntireFile(quantizedVectorData); + } + + @Override + public FloatVectorValues getFloatVectorValues(String field) throws IOException { + FieldEntry fi = fields.get(field); + if (fi == null) { + return null; + } + if (fi.vectorEncoding != VectorEncoding.FLOAT32) { + throw new IllegalArgumentException( + "field=\"" + field + "\" is encoded as: " + fi.vectorEncoding + " expected: " + VectorEncoding.FLOAT32 + ); + } + OffHeapBinarizedVectorValues bvv = OffHeapBinarizedVectorValues.load( + fi.ordToDocDISIReaderConfiguration, + fi.dimension, + fi.size, + new OptimizedScalarQuantizer(fi.similarityFunction), + fi.similarityFunction, + vectorScorer, + fi.centroid, + fi.centroidDP, + fi.vectorDataOffset, + fi.vectorDataLength, + quantizedVectorData + ); + return new BinarizedVectorValues(rawVectorsReader.getFloatVectorValues(field), bvv); + } + + @Override + public ByteVectorValues getByteVectorValues(String field) throws IOException { + return rawVectorsReader.getByteVectorValues(field); + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + rawVectorsReader.search(field, target, knnCollector, acceptDocs); + } + + @Override + public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + if (knnCollector.k() == 0) return; + final RandomVectorScorer scorer = getRandomVectorScorer(field, target); + if (scorer == null) return; + OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); + Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs); + for (int i = 0; i < scorer.maxOrd(); i++) { + if (acceptedOrds == null || acceptedOrds.get(i)) { + collector.collect(i, scorer.score(i)); + collector.incVisitedCount(1); + } + } + } + + @Override + public void close() throws IOException { + IOUtils.close(quantizedVectorData, rawVectorsReader); + } + + @Override + public long ramBytesUsed() { + long size = SHALLOW_SIZE; + size += RamUsageEstimator.sizeOfMap(fields, RamUsageEstimator.shallowSizeOfInstance(FieldEntry.class)); + size += rawVectorsReader.ramBytesUsed(); + return size; + } + + public float[] getCentroid(String field) { + FieldEntry fieldEntry = fields.get(field); + if (fieldEntry != null) { + return fieldEntry.centroid; + } + return null; + } + + private static IndexInput openDataInput( + SegmentReadState state, + int versionMeta, + String fileExtension, + String codecName, + IOContext context + ) throws IOException { + String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, fileExtension); + IndexInput in = state.directory.openInput(fileName, context); + boolean success = false; + try { + int versionVectorData = CodecUtil.checkIndexHeader( + in, + codecName, + ES818BinaryQuantizedVectorsFormat.VERSION_START, + ES818BinaryQuantizedVectorsFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + if (versionMeta != versionVectorData) { + throw new CorruptIndexException( + "Format versions mismatch: meta=" + versionMeta + ", " + codecName + "=" + versionVectorData, + in + ); + } + CodecUtil.retrieveChecksum(in); + success = true; + return in; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(in); + } + } + } + + private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException { + VectorEncoding vectorEncoding = readVectorEncoding(input); + VectorSimilarityFunction similarityFunction = readSimilarityFunction(input); + if (similarityFunction != info.getVectorSimilarityFunction()) { + throw new IllegalStateException( + "Inconsistent vector similarity function for field=\"" + + info.name + + "\"; " + + similarityFunction + + " != " + + info.getVectorSimilarityFunction() + ); + } + return FieldEntry.create(input, vectorEncoding, info.getVectorSimilarityFunction()); + } + + private record FieldEntry( + VectorSimilarityFunction similarityFunction, + VectorEncoding vectorEncoding, + int dimension, + int descritizedDimension, + long vectorDataOffset, + long vectorDataLength, + int size, + float[] centroid, + float centroidDP, + OrdToDocDISIReaderConfiguration ordToDocDISIReaderConfiguration + ) { + + static FieldEntry create(IndexInput input, VectorEncoding vectorEncoding, VectorSimilarityFunction similarityFunction) + throws IOException { + int dimension = input.readVInt(); + long vectorDataOffset = input.readVLong(); + long vectorDataLength = input.readVLong(); + int size = input.readVInt(); + final float[] centroid; + float centroidDP = 0; + if (size > 0) { + centroid = new float[dimension]; + input.readFloats(centroid, 0, dimension); + centroidDP = Float.intBitsToFloat(input.readInt()); + } else { + centroid = null; + } + OrdToDocDISIReaderConfiguration conf = OrdToDocDISIReaderConfiguration.fromStoredMeta(input, size); + return new FieldEntry( + similarityFunction, + vectorEncoding, + dimension, + BQVectorUtils.discretize(dimension, 64), + vectorDataOffset, + vectorDataLength, + size, + centroid, + centroidDP, + conf + ); + } + } + + /** Binarized vector values holding row and quantized vector values */ + protected static final class BinarizedVectorValues extends FloatVectorValues { + private final FloatVectorValues rawVectorValues; + private final BinarizedByteVectorValues quantizedVectorValues; + + BinarizedVectorValues(FloatVectorValues rawVectorValues, BinarizedByteVectorValues quantizedVectorValues) { + this.rawVectorValues = rawVectorValues; + this.quantizedVectorValues = quantizedVectorValues; + } + + @Override + public int dimension() { + return rawVectorValues.dimension(); + } + + @Override + public int size() { + return rawVectorValues.size(); + } + + @Override + public float[] vectorValue(int ord) throws IOException { + return rawVectorValues.vectorValue(ord); + } + + @Override + public BinarizedVectorValues copy() throws IOException { + return new BinarizedVectorValues(rawVectorValues.copy(), quantizedVectorValues.copy()); + } + + @Override + public Bits getAcceptOrds(Bits acceptDocs) { + return rawVectorValues.getAcceptOrds(acceptDocs); + } + + @Override + public int ordToDoc(int ord) { + return rawVectorValues.ordToDoc(ord); + } + + @Override + public DocIndexIterator iterator() { + return rawVectorValues.iterator(); + } + + @Override + public VectorScorer scorer(float[] query) throws IOException { + return quantizedVectorValues.scorer(query); + } + + BinarizedByteVectorValues getQuantizedVectorValues() throws IOException { + return quantizedVectorValues; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsWriter.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsWriter.java new file mode 100644 index 0000000000000..02dda6a4a9da1 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsWriter.java @@ -0,0 +1,944 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors.es818; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Sorter; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.internal.hppc.FloatArrayList; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.VectorScorer; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.VectorUtil; +import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier; +import org.apache.lucene.util.hnsw.RandomVectorScorer; +import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.index.codec.vectors.BQSpaceUtils; +import org.elasticsearch.index.codec.vectors.BQVectorUtils; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.apache.lucene.index.VectorSimilarityFunction.COSINE; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; +import static org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat.BINARIZED_VECTOR_COMPONENT; +import static org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + */ +@SuppressForbidden(reason = "Lucene classes") +public class ES818BinaryQuantizedVectorsWriter extends FlatVectorsWriter { + private static final long SHALLOW_RAM_BYTES_USED = shallowSizeOfInstance(ES818BinaryQuantizedVectorsWriter.class); + + private final SegmentWriteState segmentWriteState; + private final List fields = new ArrayList<>(); + private final IndexOutput meta, binarizedVectorData; + private final FlatVectorsWriter rawVectorDelegate; + private final ES818BinaryFlatVectorsScorer vectorsScorer; + private boolean finished; + + /** + * Sole constructor + * + * @param vectorsScorer the scorer to use for scoring vectors + */ + protected ES818BinaryQuantizedVectorsWriter( + ES818BinaryFlatVectorsScorer vectorsScorer, + FlatVectorsWriter rawVectorDelegate, + SegmentWriteState state + ) throws IOException { + super(vectorsScorer); + this.vectorsScorer = vectorsScorer; + this.segmentWriteState = state; + String metaFileName = IndexFileNames.segmentFileName( + state.segmentInfo.name, + state.segmentSuffix, + ES818BinaryQuantizedVectorsFormat.META_EXTENSION + ); + + String binarizedVectorDataFileName = IndexFileNames.segmentFileName( + state.segmentInfo.name, + state.segmentSuffix, + ES818BinaryQuantizedVectorsFormat.VECTOR_DATA_EXTENSION + ); + this.rawVectorDelegate = rawVectorDelegate; + boolean success = false; + try { + meta = state.directory.createOutput(metaFileName, state.context); + binarizedVectorData = state.directory.createOutput(binarizedVectorDataFileName, state.context); + + CodecUtil.writeIndexHeader( + meta, + ES818BinaryQuantizedVectorsFormat.META_CODEC_NAME, + ES818BinaryQuantizedVectorsFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + CodecUtil.writeIndexHeader( + binarizedVectorData, + ES818BinaryQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME, + ES818BinaryQuantizedVectorsFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + @Override + public FlatFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { + FlatFieldVectorsWriter rawVectorDelegate = this.rawVectorDelegate.addField(fieldInfo); + if (fieldInfo.getVectorEncoding().equals(VectorEncoding.FLOAT32)) { + @SuppressWarnings("unchecked") + FieldWriter fieldWriter = new FieldWriter(fieldInfo, (FlatFieldVectorsWriter) rawVectorDelegate); + fields.add(fieldWriter); + return fieldWriter; + } + return rawVectorDelegate; + } + + @Override + public void flush(int maxDoc, Sorter.DocMap sortMap) throws IOException { + rawVectorDelegate.flush(maxDoc, sortMap); + for (FieldWriter field : fields) { + // after raw vectors are written, normalize vectors for clustering and quantization + if (VectorSimilarityFunction.COSINE == field.fieldInfo.getVectorSimilarityFunction()) { + field.normalizeVectors(); + } + final float[] clusterCenter; + int vectorCount = field.flatFieldVectorsWriter.getVectors().size(); + clusterCenter = new float[field.dimensionSums.length]; + if (vectorCount > 0) { + for (int i = 0; i < field.dimensionSums.length; i++) { + clusterCenter[i] = field.dimensionSums[i] / vectorCount; + } + if (VectorSimilarityFunction.COSINE == field.fieldInfo.getVectorSimilarityFunction()) { + VectorUtil.l2normalize(clusterCenter); + } + } + if (segmentWriteState.infoStream.isEnabled(BINARIZED_VECTOR_COMPONENT)) { + segmentWriteState.infoStream.message(BINARIZED_VECTOR_COMPONENT, "Vectors' count:" + vectorCount); + } + OptimizedScalarQuantizer quantizer = new OptimizedScalarQuantizer(field.fieldInfo.getVectorSimilarityFunction()); + if (sortMap == null) { + writeField(field, clusterCenter, maxDoc, quantizer); + } else { + writeSortingField(field, clusterCenter, maxDoc, sortMap, quantizer); + } + field.finish(); + } + } + + private void writeField(FieldWriter fieldData, float[] clusterCenter, int maxDoc, OptimizedScalarQuantizer quantizer) + throws IOException { + // write vector values + long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); + writeBinarizedVectors(fieldData, clusterCenter, quantizer); + long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset; + float centroidDp = fieldData.getVectors().size() > 0 ? VectorUtil.dotProduct(clusterCenter, clusterCenter) : 0; + + writeMeta( + fieldData.fieldInfo, + maxDoc, + vectorDataOffset, + vectorDataLength, + clusterCenter, + centroidDp, + fieldData.getDocsWithFieldSet() + ); + } + + private void writeBinarizedVectors(FieldWriter fieldData, float[] clusterCenter, OptimizedScalarQuantizer scalarQuantizer) + throws IOException { + int discreteDims = BQVectorUtils.discretize(fieldData.fieldInfo.getVectorDimension(), 64); + byte[] quantizationScratch = new byte[discreteDims]; + byte[] vector = new byte[discreteDims / 8]; + for (int i = 0; i < fieldData.getVectors().size(); i++) { + float[] v = fieldData.getVectors().get(i); + OptimizedScalarQuantizer.QuantizationResult corrections = scalarQuantizer.scalarQuantize( + v, + quantizationScratch, + (byte) 1, + clusterCenter + ); + BQVectorUtils.packAsBinary(quantizationScratch, vector); + binarizedVectorData.writeBytes(vector, vector.length); + binarizedVectorData.writeInt(Float.floatToIntBits(corrections.lowerInterval())); + binarizedVectorData.writeInt(Float.floatToIntBits(corrections.upperInterval())); + binarizedVectorData.writeInt(Float.floatToIntBits(corrections.additionalCorrection())); + assert corrections.quantizedComponentSum() >= 0 && corrections.quantizedComponentSum() <= 0xffff; + binarizedVectorData.writeShort((short) corrections.quantizedComponentSum()); + } + } + + private void writeSortingField( + FieldWriter fieldData, + float[] clusterCenter, + int maxDoc, + Sorter.DocMap sortMap, + OptimizedScalarQuantizer scalarQuantizer + ) throws IOException { + final int[] ordMap = new int[fieldData.getDocsWithFieldSet().cardinality()]; // new ord to old ord + + DocsWithFieldSet newDocsWithField = new DocsWithFieldSet(); + mapOldOrdToNewOrd(fieldData.getDocsWithFieldSet(), sortMap, null, ordMap, newDocsWithField); + + // write vector values + long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); + writeSortedBinarizedVectors(fieldData, clusterCenter, ordMap, scalarQuantizer); + long quantizedVectorLength = binarizedVectorData.getFilePointer() - vectorDataOffset; + + float centroidDp = VectorUtil.dotProduct(clusterCenter, clusterCenter); + writeMeta(fieldData.fieldInfo, maxDoc, vectorDataOffset, quantizedVectorLength, clusterCenter, centroidDp, newDocsWithField); + } + + private void writeSortedBinarizedVectors( + FieldWriter fieldData, + float[] clusterCenter, + int[] ordMap, + OptimizedScalarQuantizer scalarQuantizer + ) throws IOException { + int discreteDims = BQVectorUtils.discretize(fieldData.fieldInfo.getVectorDimension(), 64); + byte[] quantizationScratch = new byte[discreteDims]; + byte[] vector = new byte[discreteDims / 8]; + for (int ordinal : ordMap) { + float[] v = fieldData.getVectors().get(ordinal); + OptimizedScalarQuantizer.QuantizationResult corrections = scalarQuantizer.scalarQuantize( + v, + quantizationScratch, + (byte) 1, + clusterCenter + ); + BQVectorUtils.packAsBinary(quantizationScratch, vector); + binarizedVectorData.writeBytes(vector, vector.length); + binarizedVectorData.writeInt(Float.floatToIntBits(corrections.lowerInterval())); + binarizedVectorData.writeInt(Float.floatToIntBits(corrections.upperInterval())); + binarizedVectorData.writeInt(Float.floatToIntBits(corrections.additionalCorrection())); + assert corrections.quantizedComponentSum() >= 0 && corrections.quantizedComponentSum() <= 0xffff; + binarizedVectorData.writeShort((short) corrections.quantizedComponentSum()); + } + } + + private void writeMeta( + FieldInfo field, + int maxDoc, + long vectorDataOffset, + long vectorDataLength, + float[] clusterCenter, + float centroidDp, + DocsWithFieldSet docsWithField + ) throws IOException { + meta.writeInt(field.number); + meta.writeInt(field.getVectorEncoding().ordinal()); + meta.writeInt(field.getVectorSimilarityFunction().ordinal()); + meta.writeVInt(field.getVectorDimension()); + meta.writeVLong(vectorDataOffset); + meta.writeVLong(vectorDataLength); + int count = docsWithField.cardinality(); + meta.writeVInt(count); + if (count > 0) { + final ByteBuffer buffer = ByteBuffer.allocate(field.getVectorDimension() * Float.BYTES).order(ByteOrder.LITTLE_ENDIAN); + buffer.asFloatBuffer().put(clusterCenter); + meta.writeBytes(buffer.array(), buffer.array().length); + meta.writeInt(Float.floatToIntBits(centroidDp)); + } + OrdToDocDISIReaderConfiguration.writeStoredMeta( + DIRECT_MONOTONIC_BLOCK_SHIFT, + meta, + binarizedVectorData, + count, + maxDoc, + docsWithField + ); + } + + @Override + public void finish() throws IOException { + if (finished) { + throw new IllegalStateException("already finished"); + } + finished = true; + rawVectorDelegate.finish(); + if (meta != null) { + // write end of fields marker + meta.writeInt(-1); + CodecUtil.writeFooter(meta); + } + if (binarizedVectorData != null) { + CodecUtil.writeFooter(binarizedVectorData); + } + } + + @Override + public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + if (fieldInfo.getVectorEncoding().equals(VectorEncoding.FLOAT32)) { + final float[] centroid; + final float[] mergedCentroid = new float[fieldInfo.getVectorDimension()]; + int vectorCount = mergeAndRecalculateCentroids(mergeState, fieldInfo, mergedCentroid); + // Don't need access to the random vectors, we can just use the merged + rawVectorDelegate.mergeOneField(fieldInfo, mergeState); + centroid = mergedCentroid; + if (segmentWriteState.infoStream.isEnabled(BINARIZED_VECTOR_COMPONENT)) { + segmentWriteState.infoStream.message(BINARIZED_VECTOR_COMPONENT, "Vectors' count:" + vectorCount); + } + FloatVectorValues floatVectorValues = KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); + if (fieldInfo.getVectorSimilarityFunction() == COSINE) { + floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues); + } + BinarizedFloatVectorValues binarizedVectorValues = new BinarizedFloatVectorValues( + floatVectorValues, + new OptimizedScalarQuantizer(fieldInfo.getVectorSimilarityFunction()), + centroid + ); + long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); + DocsWithFieldSet docsWithField = writeBinarizedVectorData(binarizedVectorData, binarizedVectorValues); + long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset; + float centroidDp = docsWithField.cardinality() > 0 ? VectorUtil.dotProduct(centroid, centroid) : 0; + writeMeta( + fieldInfo, + segmentWriteState.segmentInfo.maxDoc(), + vectorDataOffset, + vectorDataLength, + centroid, + centroidDp, + docsWithField + ); + } else { + rawVectorDelegate.mergeOneField(fieldInfo, mergeState); + } + } + + static DocsWithFieldSet writeBinarizedVectorAndQueryData( + IndexOutput binarizedVectorData, + IndexOutput binarizedQueryData, + FloatVectorValues floatVectorValues, + float[] centroid, + OptimizedScalarQuantizer binaryQuantizer + ) throws IOException { + int discretizedDimension = BQVectorUtils.discretize(floatVectorValues.dimension(), 64); + DocsWithFieldSet docsWithField = new DocsWithFieldSet(); + byte[][] quantizationScratch = new byte[2][floatVectorValues.dimension()]; + byte[] toIndex = new byte[discretizedDimension / 8]; + byte[] toQuery = new byte[(discretizedDimension / 8) * BQSpaceUtils.B_QUERY]; + KnnVectorValues.DocIndexIterator iterator = floatVectorValues.iterator(); + for (int docV = iterator.nextDoc(); docV != NO_MORE_DOCS; docV = iterator.nextDoc()) { + // write index vector + OptimizedScalarQuantizer.QuantizationResult[] r = binaryQuantizer.multiScalarQuantize( + floatVectorValues.vectorValue(iterator.index()), + quantizationScratch, + new byte[] { 1, 4 }, + centroid + ); + // pack and store document bit vector + BQVectorUtils.packAsBinary(quantizationScratch[0], toIndex); + binarizedVectorData.writeBytes(toIndex, toIndex.length); + binarizedVectorData.writeInt(Float.floatToIntBits(r[0].lowerInterval())); + binarizedVectorData.writeInt(Float.floatToIntBits(r[0].upperInterval())); + binarizedVectorData.writeInt(Float.floatToIntBits(r[0].additionalCorrection())); + assert r[0].quantizedComponentSum() >= 0 && r[0].quantizedComponentSum() <= 0xffff; + binarizedVectorData.writeShort((short) r[0].quantizedComponentSum()); + docsWithField.add(docV); + + // pack and store the 4bit query vector + BQSpaceUtils.transposeHalfByte(quantizationScratch[1], toQuery); + binarizedQueryData.writeBytes(toQuery, toQuery.length); + binarizedQueryData.writeInt(Float.floatToIntBits(r[1].lowerInterval())); + binarizedQueryData.writeInt(Float.floatToIntBits(r[1].upperInterval())); + binarizedQueryData.writeInt(Float.floatToIntBits(r[1].additionalCorrection())); + assert r[1].quantizedComponentSum() >= 0 && r[1].quantizedComponentSum() <= 0xffff; + binarizedQueryData.writeShort((short) r[1].quantizedComponentSum()); + } + return docsWithField; + } + + static DocsWithFieldSet writeBinarizedVectorData(IndexOutput output, BinarizedByteVectorValues binarizedByteVectorValues) + throws IOException { + DocsWithFieldSet docsWithField = new DocsWithFieldSet(); + KnnVectorValues.DocIndexIterator iterator = binarizedByteVectorValues.iterator(); + for (int docV = iterator.nextDoc(); docV != NO_MORE_DOCS; docV = iterator.nextDoc()) { + // write vector + byte[] binaryValue = binarizedByteVectorValues.vectorValue(iterator.index()); + output.writeBytes(binaryValue, binaryValue.length); + OptimizedScalarQuantizer.QuantizationResult corrections = binarizedByteVectorValues.getCorrectiveTerms(iterator.index()); + output.writeInt(Float.floatToIntBits(corrections.lowerInterval())); + output.writeInt(Float.floatToIntBits(corrections.upperInterval())); + output.writeInt(Float.floatToIntBits(corrections.additionalCorrection())); + assert corrections.quantizedComponentSum() >= 0 && corrections.quantizedComponentSum() <= 0xffff; + output.writeShort((short) corrections.quantizedComponentSum()); + docsWithField.add(docV); + } + return docsWithField; + } + + @Override + public CloseableRandomVectorScorerSupplier mergeOneFieldToIndex(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + if (fieldInfo.getVectorEncoding().equals(VectorEncoding.FLOAT32)) { + final float[] centroid; + final float cDotC; + final float[] mergedCentroid = new float[fieldInfo.getVectorDimension()]; + int vectorCount = mergeAndRecalculateCentroids(mergeState, fieldInfo, mergedCentroid); + + // Don't need access to the random vectors, we can just use the merged + rawVectorDelegate.mergeOneField(fieldInfo, mergeState); + centroid = mergedCentroid; + cDotC = vectorCount > 0 ? VectorUtil.dotProduct(centroid, centroid) : 0; + if (segmentWriteState.infoStream.isEnabled(BINARIZED_VECTOR_COMPONENT)) { + segmentWriteState.infoStream.message(BINARIZED_VECTOR_COMPONENT, "Vectors' count:" + vectorCount); + } + return mergeOneFieldToIndex(segmentWriteState, fieldInfo, mergeState, centroid, cDotC); + } + return rawVectorDelegate.mergeOneFieldToIndex(fieldInfo, mergeState); + } + + private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( + SegmentWriteState segmentWriteState, + FieldInfo fieldInfo, + MergeState mergeState, + float[] centroid, + float cDotC + ) throws IOException { + long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); + final IndexOutput tempQuantizedVectorData = segmentWriteState.directory.createTempOutput( + binarizedVectorData.getName(), + "temp", + segmentWriteState.context + ); + final IndexOutput tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput( + binarizedVectorData.getName(), + "score_temp", + segmentWriteState.context + ); + IndexInput binarizedDataInput = null; + IndexInput binarizedScoreDataInput = null; + boolean success = false; + OptimizedScalarQuantizer quantizer = new OptimizedScalarQuantizer(fieldInfo.getVectorSimilarityFunction()); + try { + FloatVectorValues floatVectorValues = KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); + if (fieldInfo.getVectorSimilarityFunction() == COSINE) { + floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues); + } + DocsWithFieldSet docsWithField = writeBinarizedVectorAndQueryData( + tempQuantizedVectorData, + tempScoreQuantizedVectorData, + floatVectorValues, + centroid, + quantizer + ); + CodecUtil.writeFooter(tempQuantizedVectorData); + IOUtils.close(tempQuantizedVectorData); + binarizedDataInput = segmentWriteState.directory.openInput(tempQuantizedVectorData.getName(), segmentWriteState.context); + binarizedVectorData.copyBytes(binarizedDataInput, binarizedDataInput.length() - CodecUtil.footerLength()); + long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset; + CodecUtil.retrieveChecksum(binarizedDataInput); + CodecUtil.writeFooter(tempScoreQuantizedVectorData); + IOUtils.close(tempScoreQuantizedVectorData); + binarizedScoreDataInput = segmentWriteState.directory.openInput( + tempScoreQuantizedVectorData.getName(), + segmentWriteState.context + ); + writeMeta( + fieldInfo, + segmentWriteState.segmentInfo.maxDoc(), + vectorDataOffset, + vectorDataLength, + centroid, + cDotC, + docsWithField + ); + success = true; + final IndexInput finalBinarizedDataInput = binarizedDataInput; + final IndexInput finalBinarizedScoreDataInput = binarizedScoreDataInput; + OffHeapBinarizedVectorValues vectorValues = new OffHeapBinarizedVectorValues.DenseOffHeapVectorValues( + fieldInfo.getVectorDimension(), + docsWithField.cardinality(), + centroid, + cDotC, + quantizer, + fieldInfo.getVectorSimilarityFunction(), + vectorsScorer, + finalBinarizedDataInput + ); + RandomVectorScorerSupplier scorerSupplier = vectorsScorer.getRandomVectorScorerSupplier( + fieldInfo.getVectorSimilarityFunction(), + new OffHeapBinarizedQueryVectorValues( + finalBinarizedScoreDataInput, + fieldInfo.getVectorDimension(), + docsWithField.cardinality() + ), + vectorValues + ); + return new BinarizedCloseableRandomVectorScorerSupplier(scorerSupplier, vectorValues, () -> { + IOUtils.close(finalBinarizedDataInput, finalBinarizedScoreDataInput); + IOUtils.deleteFilesIgnoringExceptions( + segmentWriteState.directory, + tempQuantizedVectorData.getName(), + tempScoreQuantizedVectorData.getName() + ); + }); + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException( + tempQuantizedVectorData, + tempScoreQuantizedVectorData, + binarizedDataInput, + binarizedScoreDataInput + ); + IOUtils.deleteFilesIgnoringExceptions( + segmentWriteState.directory, + tempQuantizedVectorData.getName(), + tempScoreQuantizedVectorData.getName() + ); + } + } + } + + @Override + public void close() throws IOException { + IOUtils.close(meta, binarizedVectorData, rawVectorDelegate); + } + + static float[] getCentroid(KnnVectorsReader vectorsReader, String fieldName) { + if (vectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader candidateReader) { + vectorsReader = candidateReader.getFieldReader(fieldName); + } + if (vectorsReader instanceof ES818BinaryQuantizedVectorsReader reader) { + return reader.getCentroid(fieldName); + } + return null; + } + + static int mergeAndRecalculateCentroids(MergeState mergeState, FieldInfo fieldInfo, float[] mergedCentroid) throws IOException { + boolean recalculate = false; + int totalVectorCount = 0; + for (int i = 0; i < mergeState.knnVectorsReaders.length; i++) { + KnnVectorsReader knnVectorsReader = mergeState.knnVectorsReaders[i]; + if (knnVectorsReader == null || knnVectorsReader.getFloatVectorValues(fieldInfo.name) == null) { + continue; + } + float[] centroid = getCentroid(knnVectorsReader, fieldInfo.name); + int vectorCount = knnVectorsReader.getFloatVectorValues(fieldInfo.name).size(); + if (vectorCount == 0) { + continue; + } + totalVectorCount += vectorCount; + // If there aren't centroids, or previously clustered with more than one cluster + // or if there are deleted docs, we must recalculate the centroid + if (centroid == null || mergeState.liveDocs[i] != null) { + recalculate = true; + break; + } + for (int j = 0; j < centroid.length; j++) { + mergedCentroid[j] += centroid[j] * vectorCount; + } + } + if (recalculate) { + return calculateCentroid(mergeState, fieldInfo, mergedCentroid); + } else { + for (int j = 0; j < mergedCentroid.length; j++) { + mergedCentroid[j] = mergedCentroid[j] / totalVectorCount; + } + if (fieldInfo.getVectorSimilarityFunction() == COSINE) { + VectorUtil.l2normalize(mergedCentroid); + } + return totalVectorCount; + } + } + + static int calculateCentroid(MergeState mergeState, FieldInfo fieldInfo, float[] centroid) throws IOException { + assert fieldInfo.getVectorEncoding().equals(VectorEncoding.FLOAT32); + // clear out the centroid + Arrays.fill(centroid, 0); + int count = 0; + for (int i = 0; i < mergeState.knnVectorsReaders.length; i++) { + KnnVectorsReader knnVectorsReader = mergeState.knnVectorsReaders[i]; + if (knnVectorsReader == null) continue; + FloatVectorValues vectorValues = mergeState.knnVectorsReaders[i].getFloatVectorValues(fieldInfo.name); + if (vectorValues == null) { + continue; + } + KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator(); + for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) { + ++count; + float[] vector = vectorValues.vectorValue(iterator.index()); + // TODO Panama sum + for (int j = 0; j < vector.length; j++) { + centroid[j] += vector[j]; + } + } + } + if (count == 0) { + return count; + } + // TODO Panama div + for (int i = 0; i < centroid.length; i++) { + centroid[i] /= count; + } + if (fieldInfo.getVectorSimilarityFunction() == COSINE) { + VectorUtil.l2normalize(centroid); + } + return count; + } + + @Override + public long ramBytesUsed() { + long total = SHALLOW_RAM_BYTES_USED; + for (FieldWriter field : fields) { + // the field tracks the delegate field usage + total += field.ramBytesUsed(); + } + return total; + } + + static class FieldWriter extends FlatFieldVectorsWriter { + private static final long SHALLOW_SIZE = shallowSizeOfInstance(FieldWriter.class); + private final FieldInfo fieldInfo; + private boolean finished; + private final FlatFieldVectorsWriter flatFieldVectorsWriter; + private final float[] dimensionSums; + private final FloatArrayList magnitudes = new FloatArrayList(); + + FieldWriter(FieldInfo fieldInfo, FlatFieldVectorsWriter flatFieldVectorsWriter) { + this.fieldInfo = fieldInfo; + this.flatFieldVectorsWriter = flatFieldVectorsWriter; + this.dimensionSums = new float[fieldInfo.getVectorDimension()]; + } + + @Override + public List getVectors() { + return flatFieldVectorsWriter.getVectors(); + } + + public void normalizeVectors() { + for (int i = 0; i < flatFieldVectorsWriter.getVectors().size(); i++) { + float[] vector = flatFieldVectorsWriter.getVectors().get(i); + float magnitude = magnitudes.get(i); + for (int j = 0; j < vector.length; j++) { + vector[j] /= magnitude; + } + } + } + + @Override + public DocsWithFieldSet getDocsWithFieldSet() { + return flatFieldVectorsWriter.getDocsWithFieldSet(); + } + + @Override + public void finish() throws IOException { + if (finished) { + return; + } + assert flatFieldVectorsWriter.isFinished(); + finished = true; + } + + @Override + public boolean isFinished() { + return finished && flatFieldVectorsWriter.isFinished(); + } + + @Override + public void addValue(int docID, float[] vectorValue) throws IOException { + flatFieldVectorsWriter.addValue(docID, vectorValue); + if (fieldInfo.getVectorSimilarityFunction() == COSINE) { + float dp = VectorUtil.dotProduct(vectorValue, vectorValue); + float divisor = (float) Math.sqrt(dp); + magnitudes.add(divisor); + for (int i = 0; i < vectorValue.length; i++) { + dimensionSums[i] += (vectorValue[i] / divisor); + } + } else { + for (int i = 0; i < vectorValue.length; i++) { + dimensionSums[i] += vectorValue[i]; + } + } + } + + @Override + public float[] copyValue(float[] vectorValue) { + throw new UnsupportedOperationException(); + } + + @Override + public long ramBytesUsed() { + long size = SHALLOW_SIZE; + size += flatFieldVectorsWriter.ramBytesUsed(); + size += magnitudes.ramBytesUsed(); + return size; + } + } + + // When accessing vectorValue method, targerOrd here means a row ordinal. + static class OffHeapBinarizedQueryVectorValues { + private final IndexInput slice; + private final int dimension; + private final int size; + protected final byte[] binaryValue; + protected final ByteBuffer byteBuffer; + private final int byteSize; + protected final float[] correctiveValues; + private int lastOrd = -1; + private int quantizedComponentSum; + + OffHeapBinarizedQueryVectorValues(IndexInput data, int dimension, int size) { + this.slice = data; + this.dimension = dimension; + this.size = size; + // 4x the quantized binary dimensions + int binaryDimensions = (BQVectorUtils.discretize(dimension, 64) / 8) * BQSpaceUtils.B_QUERY; + this.byteBuffer = ByteBuffer.allocate(binaryDimensions); + this.binaryValue = byteBuffer.array(); + // + 1 for the quantized sum + this.correctiveValues = new float[3]; + this.byteSize = binaryDimensions + Float.BYTES * 3 + Short.BYTES; + } + + public OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return new OptimizedScalarQuantizer.QuantizationResult( + correctiveValues[0], + correctiveValues[1], + correctiveValues[2], + quantizedComponentSum + ); + } + vectorValue(targetOrd); + return new OptimizedScalarQuantizer.QuantizationResult( + correctiveValues[0], + correctiveValues[1], + correctiveValues[2], + quantizedComponentSum + ); + } + + public int size() { + return size; + } + + public int dimension() { + return dimension; + } + + public OffHeapBinarizedQueryVectorValues copy() throws IOException { + return new OffHeapBinarizedQueryVectorValues(slice.clone(), dimension, size); + } + + public IndexInput getSlice() { + return slice; + } + + public byte[] vectorValue(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return binaryValue; + } + slice.seek((long) targetOrd * byteSize); + slice.readBytes(binaryValue, 0, binaryValue.length); + slice.readFloats(correctiveValues, 0, 3); + quantizedComponentSum = Short.toUnsignedInt(slice.readShort()); + lastOrd = targetOrd; + return binaryValue; + } + } + + static class BinarizedFloatVectorValues extends BinarizedByteVectorValues { + private OptimizedScalarQuantizer.QuantizationResult corrections; + private final byte[] binarized; + private final byte[] initQuantized; + private final float[] centroid; + private final FloatVectorValues values; + private final OptimizedScalarQuantizer quantizer; + + private int lastOrd = -1; + + BinarizedFloatVectorValues(FloatVectorValues delegate, OptimizedScalarQuantizer quantizer, float[] centroid) { + this.values = delegate; + this.quantizer = quantizer; + this.binarized = new byte[BQVectorUtils.discretize(delegate.dimension(), 64) / 8]; + this.initQuantized = new byte[delegate.dimension()]; + this.centroid = centroid; + } + + @Override + public OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int ord) { + if (ord != lastOrd) { + throw new IllegalStateException( + "attempt to retrieve corrective terms for different ord " + ord + " than the quantization was done for: " + lastOrd + ); + } + return corrections; + } + + @Override + public byte[] vectorValue(int ord) throws IOException { + if (ord != lastOrd) { + binarize(ord); + lastOrd = ord; + } + return binarized; + } + + @Override + public int dimension() { + return values.dimension(); + } + + @Override + public OptimizedScalarQuantizer getQuantizer() { + throw new UnsupportedOperationException(); + } + + @Override + public float[] getCentroid() throws IOException { + return centroid; + } + + @Override + public int size() { + return values.size(); + } + + @Override + public VectorScorer scorer(float[] target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BinarizedByteVectorValues copy() throws IOException { + return new BinarizedFloatVectorValues(values.copy(), quantizer, centroid); + } + + private void binarize(int ord) throws IOException { + corrections = quantizer.scalarQuantize(values.vectorValue(ord), initQuantized, (byte) 1, centroid); + BQVectorUtils.packAsBinary(initQuantized, binarized); + } + + @Override + public DocIndexIterator iterator() { + return values.iterator(); + } + + @Override + public int ordToDoc(int ord) { + return values.ordToDoc(ord); + } + } + + static class BinarizedCloseableRandomVectorScorerSupplier implements CloseableRandomVectorScorerSupplier { + private final RandomVectorScorerSupplier supplier; + private final KnnVectorValues vectorValues; + private final Closeable onClose; + + BinarizedCloseableRandomVectorScorerSupplier(RandomVectorScorerSupplier supplier, KnnVectorValues vectorValues, Closeable onClose) { + this.supplier = supplier; + this.onClose = onClose; + this.vectorValues = vectorValues; + } + + @Override + public RandomVectorScorer scorer(int ord) throws IOException { + return supplier.scorer(ord); + } + + @Override + public RandomVectorScorerSupplier copy() throws IOException { + return supplier.copy(); + } + + @Override + public void close() throws IOException { + onClose.close(); + } + + @Override + public int totalVectorCount() { + return vectorValues.size(); + } + } + + static final class NormalizedFloatVectorValues extends FloatVectorValues { + private final FloatVectorValues values; + private final float[] normalizedVector; + + NormalizedFloatVectorValues(FloatVectorValues values) { + this.values = values; + this.normalizedVector = new float[values.dimension()]; + } + + @Override + public int dimension() { + return values.dimension(); + } + + @Override + public int size() { + return values.size(); + } + + @Override + public int ordToDoc(int ord) { + return values.ordToDoc(ord); + } + + @Override + public float[] vectorValue(int ord) throws IOException { + System.arraycopy(values.vectorValue(ord), 0, normalizedVector, 0, normalizedVector.length); + VectorUtil.l2normalize(normalizedVector); + return normalizedVector; + } + + @Override + public DocIndexIterator iterator() { + return values.iterator(); + } + + @Override + public NormalizedFloatVectorValues copy() throws IOException { + return new NormalizedFloatVectorValues(values.copy()); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormat.java new file mode 100644 index 0000000000000..56942017c3cef --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormat.java @@ -0,0 +1,145 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors.es818; + +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.search.TaskExecutor; +import org.apache.lucene.util.hnsw.HnswGraph; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; + +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_MAX_CONN; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + */ +public class ES818HnswBinaryQuantizedVectorsFormat extends KnnVectorsFormat { + + public static final String NAME = "ES818HnswBinaryQuantizedVectorsFormat"; + + /** + * Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to + * {@link Lucene99HnswVectorsFormat#DEFAULT_MAX_CONN}. See {@link HnswGraph} for more details. + */ + private final int maxConn; + + /** + * The number of candidate neighbors to track while searching the graph for each newly inserted + * node. Defaults to {@link Lucene99HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link HnswGraph} + * for details. + */ + private final int beamWidth; + + /** The format for storing, reading, merging vectors on disk */ + private static final FlatVectorsFormat flatVectorsFormat = new ES818BinaryQuantizedVectorsFormat(); + + private final int numMergeWorkers; + private final TaskExecutor mergeExec; + + /** Constructs a format using default graph construction parameters */ + public ES818HnswBinaryQuantizedVectorsFormat() { + this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, null); + } + + /** + * Constructs a format using the given graph construction parameters. + * + * @param maxConn the maximum number of connections to a node in the HNSW graph + * @param beamWidth the size of the queue maintained during graph construction. + */ + public ES818HnswBinaryQuantizedVectorsFormat(int maxConn, int beamWidth) { + this(maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, null); + } + + /** + * Constructs a format using the given graph construction parameters and scalar quantization. + * + * @param maxConn the maximum number of connections to a node in the HNSW graph + * @param beamWidth the size of the queue maintained during graph construction. + * @param numMergeWorkers number of workers (threads) that will be used when doing merge. If + * larger than 1, a non-null {@link ExecutorService} must be passed as mergeExec + * @param mergeExec the {@link ExecutorService} that will be used by ALL vector writers that are + * generated by this format to do the merge + */ + public ES818HnswBinaryQuantizedVectorsFormat(int maxConn, int beamWidth, int numMergeWorkers, ExecutorService mergeExec) { + super(NAME); + if (maxConn <= 0 || maxConn > MAXIMUM_MAX_CONN) { + throw new IllegalArgumentException( + "maxConn must be positive and less than or equal to " + MAXIMUM_MAX_CONN + "; maxConn=" + maxConn + ); + } + if (beamWidth <= 0 || beamWidth > MAXIMUM_BEAM_WIDTH) { + throw new IllegalArgumentException( + "beamWidth must be positive and less than or equal to " + MAXIMUM_BEAM_WIDTH + "; beamWidth=" + beamWidth + ); + } + this.maxConn = maxConn; + this.beamWidth = beamWidth; + if (numMergeWorkers == 1 && mergeExec != null) { + throw new IllegalArgumentException("No executor service is needed as we'll use single thread to merge"); + } + this.numMergeWorkers = numMergeWorkers; + if (mergeExec != null) { + this.mergeExec = new TaskExecutor(mergeExec); + } else { + this.mergeExec = null; + } + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new Lucene99HnswVectorsWriter(state, maxConn, beamWidth, flatVectorsFormat.fieldsWriter(state), numMergeWorkers, mergeExec); + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state)); + } + + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } + + @Override + public String toString() { + return "ES818HnswBinaryQuantizedVectorsFormat(name=ES818HnswBinaryQuantizedVectorsFormat, maxConn=" + + maxConn + + ", beamWidth=" + + beamWidth + + ", flatVectorFormat=" + + flatVectorsFormat + + ")"; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/OffHeapBinarizedVectorValues.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/OffHeapBinarizedVectorValues.java new file mode 100644 index 0000000000000..72333169b39b5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/OffHeapBinarizedVectorValues.java @@ -0,0 +1,371 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors.es818; + +import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; +import org.apache.lucene.codecs.lucene90.IndexedDISI; +import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.VectorScorer; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.hnsw.RandomVectorScorer; +import org.apache.lucene.util.packed.DirectMonotonicReader; +import org.elasticsearch.index.codec.vectors.BQVectorUtils; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** Binarized vector values loaded from off-heap */ +abstract class OffHeapBinarizedVectorValues extends BinarizedByteVectorValues { + + final int dimension; + final int size; + final int numBytes; + final VectorSimilarityFunction similarityFunction; + final FlatVectorsScorer vectorsScorer; + + final IndexInput slice; + final byte[] binaryValue; + final ByteBuffer byteBuffer; + final int byteSize; + private int lastOrd = -1; + final float[] correctiveValues; + int quantizedComponentSum; + final OptimizedScalarQuantizer binaryQuantizer; + final float[] centroid; + final float centroidDp; + private final int discretizedDimensions; + + OffHeapBinarizedVectorValues( + int dimension, + int size, + float[] centroid, + float centroidDp, + OptimizedScalarQuantizer quantizer, + VectorSimilarityFunction similarityFunction, + FlatVectorsScorer vectorsScorer, + IndexInput slice + ) { + this.dimension = dimension; + this.size = size; + this.similarityFunction = similarityFunction; + this.vectorsScorer = vectorsScorer; + this.slice = slice; + this.centroid = centroid; + this.centroidDp = centroidDp; + this.numBytes = BQVectorUtils.discretize(dimension, 64) / 8; + this.correctiveValues = new float[3]; + this.byteSize = numBytes + (Float.BYTES * 3) + Short.BYTES; + this.byteBuffer = ByteBuffer.allocate(numBytes); + this.binaryValue = byteBuffer.array(); + this.binaryQuantizer = quantizer; + this.discretizedDimensions = BQVectorUtils.discretize(dimension, 64); + } + + @Override + public int dimension() { + return dimension; + } + + @Override + public int size() { + return size; + } + + @Override + public byte[] vectorValue(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return binaryValue; + } + slice.seek((long) targetOrd * byteSize); + slice.readBytes(byteBuffer.array(), byteBuffer.arrayOffset(), numBytes); + slice.readFloats(correctiveValues, 0, 3); + quantizedComponentSum = Short.toUnsignedInt(slice.readShort()); + lastOrd = targetOrd; + return binaryValue; + } + + @Override + public int discretizedDimensions() { + return discretizedDimensions; + } + + @Override + public float getCentroidDP() { + return centroidDp; + } + + @Override + public OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return new OptimizedScalarQuantizer.QuantizationResult( + correctiveValues[0], + correctiveValues[1], + correctiveValues[2], + quantizedComponentSum + ); + } + slice.seek(((long) targetOrd * byteSize) + numBytes); + slice.readFloats(correctiveValues, 0, 3); + quantizedComponentSum = Short.toUnsignedInt(slice.readShort()); + return new OptimizedScalarQuantizer.QuantizationResult( + correctiveValues[0], + correctiveValues[1], + correctiveValues[2], + quantizedComponentSum + ); + } + + @Override + public OptimizedScalarQuantizer getQuantizer() { + return binaryQuantizer; + } + + @Override + public float[] getCentroid() { + return centroid; + } + + @Override + public int getVectorByteLength() { + return numBytes; + } + + static OffHeapBinarizedVectorValues load( + OrdToDocDISIReaderConfiguration configuration, + int dimension, + int size, + OptimizedScalarQuantizer binaryQuantizer, + VectorSimilarityFunction similarityFunction, + FlatVectorsScorer vectorsScorer, + float[] centroid, + float centroidDp, + long quantizedVectorDataOffset, + long quantizedVectorDataLength, + IndexInput vectorData + ) throws IOException { + if (configuration.isEmpty()) { + return new EmptyOffHeapVectorValues(dimension, similarityFunction, vectorsScorer); + } + assert centroid != null; + IndexInput bytesSlice = vectorData.slice("quantized-vector-data", quantizedVectorDataOffset, quantizedVectorDataLength); + if (configuration.isDense()) { + return new DenseOffHeapVectorValues( + dimension, + size, + centroid, + centroidDp, + binaryQuantizer, + similarityFunction, + vectorsScorer, + bytesSlice + ); + } else { + return new SparseOffHeapVectorValues( + configuration, + dimension, + size, + centroid, + centroidDp, + binaryQuantizer, + vectorData, + similarityFunction, + vectorsScorer, + bytesSlice + ); + } + } + + /** Dense off-heap binarized vector values */ + static class DenseOffHeapVectorValues extends OffHeapBinarizedVectorValues { + DenseOffHeapVectorValues( + int dimension, + int size, + float[] centroid, + float centroidDp, + OptimizedScalarQuantizer binaryQuantizer, + VectorSimilarityFunction similarityFunction, + FlatVectorsScorer vectorsScorer, + IndexInput slice + ) { + super(dimension, size, centroid, centroidDp, binaryQuantizer, similarityFunction, vectorsScorer, slice); + } + + @Override + public DenseOffHeapVectorValues copy() throws IOException { + return new DenseOffHeapVectorValues( + dimension, + size, + centroid, + centroidDp, + binaryQuantizer, + similarityFunction, + vectorsScorer, + slice.clone() + ); + } + + @Override + public Bits getAcceptOrds(Bits acceptDocs) { + return acceptDocs; + } + + @Override + public VectorScorer scorer(float[] target) throws IOException { + DenseOffHeapVectorValues copy = copy(); + DocIndexIterator iterator = copy.iterator(); + RandomVectorScorer scorer = vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target); + return new VectorScorer() { + @Override + public float score() throws IOException { + return scorer.score(iterator.index()); + } + + @Override + public DocIdSetIterator iterator() { + return iterator; + } + }; + } + + @Override + public DocIndexIterator iterator() { + return createDenseIterator(); + } + } + + /** Sparse off-heap binarized vector values */ + private static class SparseOffHeapVectorValues extends OffHeapBinarizedVectorValues { + private final DirectMonotonicReader ordToDoc; + private final IndexedDISI disi; + // dataIn was used to init a new IndexedDIS for #randomAccess() + private final IndexInput dataIn; + private final OrdToDocDISIReaderConfiguration configuration; + + SparseOffHeapVectorValues( + OrdToDocDISIReaderConfiguration configuration, + int dimension, + int size, + float[] centroid, + float centroidDp, + OptimizedScalarQuantizer binaryQuantizer, + IndexInput dataIn, + VectorSimilarityFunction similarityFunction, + FlatVectorsScorer vectorsScorer, + IndexInput slice + ) throws IOException { + super(dimension, size, centroid, centroidDp, binaryQuantizer, similarityFunction, vectorsScorer, slice); + this.configuration = configuration; + this.dataIn = dataIn; + this.ordToDoc = configuration.getDirectMonotonicReader(dataIn); + this.disi = configuration.getIndexedDISI(dataIn); + } + + @Override + public SparseOffHeapVectorValues copy() throws IOException { + return new SparseOffHeapVectorValues( + configuration, + dimension, + size, + centroid, + centroidDp, + binaryQuantizer, + dataIn, + similarityFunction, + vectorsScorer, + slice.clone() + ); + } + + @Override + public int ordToDoc(int ord) { + return (int) ordToDoc.get(ord); + } + + @Override + public Bits getAcceptOrds(Bits acceptDocs) { + if (acceptDocs == null) { + return null; + } + return new Bits() { + @Override + public boolean get(int index) { + return acceptDocs.get(ordToDoc(index)); + } + + @Override + public int length() { + return size; + } + }; + } + + @Override + public DocIndexIterator iterator() { + return IndexedDISI.asDocIndexIterator(disi); + } + + @Override + public VectorScorer scorer(float[] target) throws IOException { + SparseOffHeapVectorValues copy = copy(); + DocIndexIterator iterator = copy.iterator(); + RandomVectorScorer scorer = vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target); + return new VectorScorer() { + @Override + public float score() throws IOException { + return scorer.score(iterator.index()); + } + + @Override + public DocIdSetIterator iterator() { + return iterator; + } + }; + } + } + + private static class EmptyOffHeapVectorValues extends OffHeapBinarizedVectorValues { + EmptyOffHeapVectorValues(int dimension, VectorSimilarityFunction similarityFunction, FlatVectorsScorer vectorsScorer) { + super(dimension, 0, null, Float.NaN, null, similarityFunction, vectorsScorer, null); + } + + @Override + public DocIndexIterator iterator() { + return createDenseIterator(); + } + + @Override + public DenseOffHeapVectorValues copy() { + throw new UnsupportedOperationException(); + } + + @Override + public Bits getAcceptOrds(Bits acceptDocs) { + return null; + } + + @Override + public VectorScorer scorer(float[] target) { + return null; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/OptimizedScalarQuantizer.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/OptimizedScalarQuantizer.java new file mode 100644 index 0000000000000..d5ed38cb5a0e1 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/OptimizedScalarQuantizer.java @@ -0,0 +1,246 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es818; + +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.util.VectorUtil; + +import static org.apache.lucene.index.VectorSimilarityFunction.COSINE; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +class OptimizedScalarQuantizer { + // The initial interval is set to the minimum MSE grid for each number of bits + // these starting points are derived from the optimal MSE grid for a uniform distribution + static final float[][] MINIMUM_MSE_GRID = new float[][] { + { -0.798f, 0.798f }, + { -1.493f, 1.493f }, + { -2.051f, 2.051f }, + { -2.514f, 2.514f }, + { -2.916f, 2.916f }, + { -3.278f, 3.278f }, + { -3.611f, 3.611f }, + { -3.922f, 3.922f } }; + private static final float DEFAULT_LAMBDA = 0.1f; + private static final int DEFAULT_ITERS = 5; + private final VectorSimilarityFunction similarityFunction; + private final float lambda; + private final int iters; + + OptimizedScalarQuantizer(VectorSimilarityFunction similarityFunction, float lambda, int iters) { + this.similarityFunction = similarityFunction; + this.lambda = lambda; + this.iters = iters; + } + + OptimizedScalarQuantizer(VectorSimilarityFunction similarityFunction) { + this(similarityFunction, DEFAULT_LAMBDA, DEFAULT_ITERS); + } + + public record QuantizationResult(float lowerInterval, float upperInterval, float additionalCorrection, int quantizedComponentSum) {} + + public QuantizationResult[] multiScalarQuantize(float[] vector, byte[][] destinations, byte[] bits, float[] centroid) { + assert similarityFunction != COSINE || VectorUtil.isUnitVector(vector); + assert similarityFunction != COSINE || VectorUtil.isUnitVector(centroid); + assert bits.length == destinations.length; + float[] intervalScratch = new float[2]; + double vecMean = 0; + double vecVar = 0; + float norm2 = 0; + float centroidDot = 0; + float min = Float.MAX_VALUE; + float max = -Float.MAX_VALUE; + for (int i = 0; i < vector.length; ++i) { + if (similarityFunction != EUCLIDEAN) { + centroidDot += vector[i] * centroid[i]; + } + vector[i] = vector[i] - centroid[i]; + min = Math.min(min, vector[i]); + max = Math.max(max, vector[i]); + norm2 += (vector[i] * vector[i]); + double delta = vector[i] - vecMean; + vecMean += delta / (i + 1); + vecVar += delta * (vector[i] - vecMean); + } + vecVar /= vector.length; + double vecStd = Math.sqrt(vecVar); + QuantizationResult[] results = new QuantizationResult[bits.length]; + for (int i = 0; i < bits.length; ++i) { + assert bits[i] > 0 && bits[i] <= 8; + int points = (1 << bits[i]); + // Linearly scale the interval to the standard deviation of the vector, ensuring we are within the min/max bounds + intervalScratch[0] = (float) clamp((MINIMUM_MSE_GRID[bits[i] - 1][0] + vecMean) * vecStd, min, max); + intervalScratch[1] = (float) clamp((MINIMUM_MSE_GRID[bits[i] - 1][1] + vecMean) * vecStd, min, max); + optimizeIntervals(intervalScratch, vector, norm2, points); + float nSteps = ((1 << bits[i]) - 1); + float a = intervalScratch[0]; + float b = intervalScratch[1]; + float step = (b - a) / nSteps; + int sumQuery = 0; + // Now we have the optimized intervals, quantize the vector + for (int h = 0; h < vector.length; h++) { + float xi = (float) clamp(vector[h], a, b); + int assignment = Math.round((xi - a) / step); + sumQuery += assignment; + destinations[i][h] = (byte) assignment; + } + results[i] = new QuantizationResult( + intervalScratch[0], + intervalScratch[1], + similarityFunction == EUCLIDEAN ? norm2 : centroidDot, + sumQuery + ); + } + return results; + } + + public QuantizationResult scalarQuantize(float[] vector, byte[] destination, byte bits, float[] centroid) { + assert similarityFunction != COSINE || VectorUtil.isUnitVector(vector); + assert similarityFunction != COSINE || VectorUtil.isUnitVector(centroid); + assert vector.length <= destination.length; + assert bits > 0 && bits <= 8; + float[] intervalScratch = new float[2]; + int points = 1 << bits; + double vecMean = 0; + double vecVar = 0; + float norm2 = 0; + float centroidDot = 0; + float min = Float.MAX_VALUE; + float max = -Float.MAX_VALUE; + for (int i = 0; i < vector.length; ++i) { + if (similarityFunction != EUCLIDEAN) { + centroidDot += vector[i] * centroid[i]; + } + vector[i] = vector[i] - centroid[i]; + min = Math.min(min, vector[i]); + max = Math.max(max, vector[i]); + norm2 += (vector[i] * vector[i]); + double delta = vector[i] - vecMean; + vecMean += delta / (i + 1); + vecVar += delta * (vector[i] - vecMean); + } + vecVar /= vector.length; + double vecStd = Math.sqrt(vecVar); + // Linearly scale the interval to the standard deviation of the vector, ensuring we are within the min/max bounds + intervalScratch[0] = (float) clamp((MINIMUM_MSE_GRID[bits - 1][0] + vecMean) * vecStd, min, max); + intervalScratch[1] = (float) clamp((MINIMUM_MSE_GRID[bits - 1][1] + vecMean) * vecStd, min, max); + optimizeIntervals(intervalScratch, vector, norm2, points); + float nSteps = ((1 << bits) - 1); + // Now we have the optimized intervals, quantize the vector + float a = intervalScratch[0]; + float b = intervalScratch[1]; + float step = (b - a) / nSteps; + int sumQuery = 0; + for (int h = 0; h < vector.length; h++) { + float xi = (float) clamp(vector[h], a, b); + int assignment = Math.round((xi - a) / step); + sumQuery += assignment; + destination[h] = (byte) assignment; + } + return new QuantizationResult( + intervalScratch[0], + intervalScratch[1], + similarityFunction == EUCLIDEAN ? norm2 : centroidDot, + sumQuery + ); + } + + /** + * Compute the loss of the vector given the interval. Effectively, we are computing the MSE of a dequantized vector with the raw + * vector. + * @param vector raw vector + * @param interval interval to quantize the vector + * @param points number of quantization points + * @param norm2 squared norm of the vector + * @return the loss + */ + private double loss(float[] vector, float[] interval, int points, float norm2) { + double a = interval[0]; + double b = interval[1]; + double step = ((b - a) / (points - 1.0F)); + double stepInv = 1.0 / step; + double xe = 0.0; + double e = 0.0; + for (double xi : vector) { + // this is quantizing and then dequantizing the vector + double xiq = (a + step * Math.round((clamp(xi, a, b) - a) * stepInv)); + // how much does the de-quantized value differ from the original value + xe += xi * (xi - xiq); + e += (xi - xiq) * (xi - xiq); + } + return (1.0 - lambda) * xe * xe / norm2 + lambda * e; + } + + /** + * Optimize the quantization interval for the given vector. This is done via a coordinate descent trying to minimize the quantization + * loss. Note, the loss is not always guaranteed to decrease, so we have a maximum number of iterations and will exit early if the + * loss increases. + * @param initInterval initial interval, the optimized interval will be stored here + * @param vector raw vector + * @param norm2 squared norm of the vector + * @param points number of quantization points + */ + private void optimizeIntervals(float[] initInterval, float[] vector, float norm2, int points) { + double initialLoss = loss(vector, initInterval, points, norm2); + final float scale = (1.0f - lambda) / norm2; + if (Float.isFinite(scale) == false) { + return; + } + for (int i = 0; i < iters; ++i) { + float a = initInterval[0]; + float b = initInterval[1]; + float stepInv = (points - 1.0f) / (b - a); + // calculate the grid points for coordinate descent + double daa = 0; + double dab = 0; + double dbb = 0; + double dax = 0; + double dbx = 0; + for (float xi : vector) { + float k = Math.round((clamp(xi, a, b) - a) * stepInv); + float s = k / (points - 1); + daa += (1.0 - s) * (1.0 - s); + dab += (1.0 - s) * s; + dbb += s * s; + dax += xi * (1.0 - s); + dbx += xi * s; + } + double m0 = scale * dax * dax + lambda * daa; + double m1 = scale * dax * dbx + lambda * dab; + double m2 = scale * dbx * dbx + lambda * dbb; + // its possible that the determinant is 0, in which case we can't update the interval + double det = m0 * m2 - m1 * m1; + if (det == 0) { + return; + } + float aOpt = (float) ((m2 * dax - m1 * dbx) / det); + float bOpt = (float) ((m0 * dbx - m1 * dax) / det); + // If there is no change in the interval, we can stop + if ((Math.abs(initInterval[0] - aOpt) < 1e-8 && Math.abs(initInterval[1] - bOpt) < 1e-8)) { + return; + } + double newLoss = loss(vector, new float[] { aOpt, bOpt }, points, norm2); + // If the new loss is worse, don't update the interval and exit + // This optimization, unlike kMeans, does not always converge to better loss + // So exit if we are getting worse + if (newLoss > initialLoss) { + return; + } + // Update the interval and go again + initInterval[0] = aOpt; + initInterval[1] = bOpt; + initialLoss = newLoss; + } + } + + private static double clamp(double x, double a, double b) { + return Math.min(Math.max(x, a), b); + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 0a6a24f727572..d780faad96f2d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -46,8 +46,8 @@ import org.elasticsearch.index.codec.vectors.ES814HnswScalarQuantizedVectorsFormat; import org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat; import org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat; -import org.elasticsearch.index.codec.vectors.es816.ES816BinaryQuantizedVectorsFormat; -import org.elasticsearch.index.codec.vectors.es816.ES816HnswBinaryQuantizedVectorsFormat; +import org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat; +import org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.ArraySourceValueFetcher; @@ -1788,7 +1788,7 @@ static class BBQHnswIndexOptions extends IndexOptions { @Override KnnVectorsFormat getVectorsFormat(ElementType elementType) { assert elementType == ElementType.FLOAT; - return new ES816HnswBinaryQuantizedVectorsFormat(m, efConstruction); + return new ES818HnswBinaryQuantizedVectorsFormat(m, efConstruction); } @Override @@ -1836,7 +1836,7 @@ static class BBQFlatIndexOptions extends IndexOptions { @Override KnnVectorsFormat getVectorsFormat(ElementType elementType) { assert elementType == ElementType.FLOAT; - return new ES816BinaryQuantizedVectorsFormat(); + return new ES818BinaryQuantizedVectorsFormat(); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index 794b30aa5aab2..57980321bdc3d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -44,6 +44,7 @@ private SearchCapabilities() {} private static final String MULTI_DENSE_VECTOR_SCRIPT_MAX_SIM = "multi_dense_vector_script_max_sim_with_bugfix"; private static final String RANDOM_SAMPLER_WITH_SCORED_SUBAGGS = "random_sampler_with_scored_subaggs"; + private static final String OPTIMIZED_SCALAR_QUANTIZATION_BBQ = "optimized_scalar_quantization_bbq"; public static final Set CAPABILITIES; static { @@ -55,6 +56,7 @@ private SearchCapabilities() {} capabilities.add(TRANSFORM_RANK_RRF_TO_RETRIEVER); capabilities.add(NESTED_RETRIEVER_INNER_HITS_SUPPORT); capabilities.add(RANDOM_SAMPLER_WITH_SCORED_SUBAGGS); + capabilities.add(OPTIMIZED_SCALAR_QUANTIZATION_BBQ); if (MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()) { capabilities.add(MULTI_DENSE_VECTOR_FIELD_MAPPER); capabilities.add(MULTI_DENSE_VECTOR_SCRIPT_ACCESS); diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index 389555e60b43b..cef8d09980814 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -5,3 +5,5 @@ org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat org.elasticsearch.index.codec.vectors.es816.ES816BinaryQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es816.ES816HnswBinaryQuantizedVectorsFormat +org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat +org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/BQVectorUtilsTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/BQVectorUtilsTests.java index 9f9114c70b6db..270ad54e9a962 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/BQVectorUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/BQVectorUtilsTests.java @@ -38,6 +38,32 @@ public static int popcount(byte[] a, int aOffset, byte[] b, int length) { private static float DELTA = Float.MIN_VALUE; + public void testPackAsBinary() { + // 5 bits + byte[] toPack = new byte[] { 1, 1, 0, 0, 1 }; + byte[] packed = new byte[1]; + BQVectorUtils.packAsBinary(toPack, packed); + assertArrayEquals(new byte[] { (byte) 0b11001000 }, packed); + + // 8 bits + toPack = new byte[] { 1, 1, 0, 0, 1, 0, 1, 0 }; + packed = new byte[1]; + BQVectorUtils.packAsBinary(toPack, packed); + assertArrayEquals(new byte[] { (byte) 0b11001010 }, packed); + + // 10 bits + toPack = new byte[] { 1, 1, 0, 0, 1, 0, 1, 0, 1, 1 }; + packed = new byte[2]; + BQVectorUtils.packAsBinary(toPack, packed); + assertArrayEquals(new byte[] { (byte) 0b11001010, (byte) 0b11000000 }, packed); + + // 16 bits + toPack = new byte[] { 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0 }; + packed = new byte[2]; + BQVectorUtils.packAsBinary(toPack, packed); + assertArrayEquals(new byte[] { (byte) 0b11001010, (byte) 0b11100110 }, packed); + } + public void testPadFloat() { assertArrayEquals(new float[] { 1, 2, 3, 4 }, BQVectorUtils.pad(new float[] { 1, 2, 3, 4 }, 4), DELTA); assertArrayEquals(new float[] { 1, 2, 3, 4 }, BQVectorUtils.pad(new float[] { 1, 2, 3, 4 }, 3), DELTA); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatRWVectorsScorer.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatRWVectorsScorer.java new file mode 100644 index 0000000000000..0bebe16f468ce --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatRWVectorsScorer.java @@ -0,0 +1,256 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors.es816; + +import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.VectorUtil; +import org.apache.lucene.util.hnsw.RandomVectorScorer; +import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier; +import org.elasticsearch.index.codec.vectors.BQSpaceUtils; +import org.elasticsearch.index.codec.vectors.BQVectorUtils; +import org.elasticsearch.simdvec.ESVectorUtil; + +import java.io.IOException; + +import static org.apache.lucene.index.VectorSimilarityFunction.COSINE; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; +import static org.apache.lucene.index.VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT; + +/** Vector scorer over binarized vector values */ +class ES816BinaryFlatRWVectorsScorer implements FlatVectorsScorer { + private final FlatVectorsScorer nonQuantizedDelegate; + + ES816BinaryFlatRWVectorsScorer(FlatVectorsScorer nonQuantizedDelegate) { + this.nonQuantizedDelegate = nonQuantizedDelegate; + } + + @Override + public RandomVectorScorerSupplier getRandomVectorScorerSupplier( + VectorSimilarityFunction similarityFunction, + KnnVectorValues vectorValues + ) throws IOException { + if (vectorValues instanceof BinarizedByteVectorValues) { + throw new UnsupportedOperationException( + "getRandomVectorScorerSupplier(VectorSimilarityFunction,RandomAccessVectorValues) not implemented for binarized format" + ); + } + return nonQuantizedDelegate.getRandomVectorScorerSupplier(similarityFunction, vectorValues); + } + + @Override + public RandomVectorScorer getRandomVectorScorer( + VectorSimilarityFunction similarityFunction, + KnnVectorValues vectorValues, + float[] target + ) throws IOException { + if (vectorValues instanceof BinarizedByteVectorValues binarizedVectors) { + BinaryQuantizer quantizer = binarizedVectors.getQuantizer(); + float[] centroid = binarizedVectors.getCentroid(); + // FIXME: precompute this once? + int discretizedDimensions = BQVectorUtils.discretize(target.length, 64); + if (similarityFunction == COSINE) { + float[] copy = ArrayUtil.copyOfSubArray(target, 0, target.length); + VectorUtil.l2normalize(copy); + target = copy; + } + byte[] quantized = new byte[BQSpaceUtils.B_QUERY * discretizedDimensions / 8]; + BinaryQuantizer.QueryFactors factors = quantizer.quantizeForQuery(target, quantized, centroid); + BinaryQueryVector queryVector = new BinaryQueryVector(quantized, factors); + return new BinarizedRandomVectorScorer(queryVector, binarizedVectors, similarityFunction); + } + return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target); + } + + @Override + public RandomVectorScorer getRandomVectorScorer( + VectorSimilarityFunction similarityFunction, + KnnVectorValues vectorValues, + byte[] target + ) throws IOException { + return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target); + } + + RandomVectorScorerSupplier getRandomVectorScorerSupplier( + VectorSimilarityFunction similarityFunction, + ES816BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues scoringVectors, + BinarizedByteVectorValues targetVectors + ) { + return new BinarizedRandomVectorScorerSupplier(scoringVectors, targetVectors, similarityFunction); + } + + @Override + public String toString() { + return "ES816BinaryFlatVectorsScorer(nonQuantizedDelegate=" + nonQuantizedDelegate + ")"; + } + + /** Vector scorer supplier over binarized vector values */ + static class BinarizedRandomVectorScorerSupplier implements RandomVectorScorerSupplier { + private final ES816BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues queryVectors; + private final BinarizedByteVectorValues targetVectors; + private final VectorSimilarityFunction similarityFunction; + + BinarizedRandomVectorScorerSupplier( + ES816BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues queryVectors, + BinarizedByteVectorValues targetVectors, + VectorSimilarityFunction similarityFunction + ) { + this.queryVectors = queryVectors; + this.targetVectors = targetVectors; + this.similarityFunction = similarityFunction; + } + + @Override + public RandomVectorScorer scorer(int ord) throws IOException { + byte[] vector = queryVectors.vectorValue(ord); + int quantizedSum = queryVectors.sumQuantizedValues(ord); + float distanceToCentroid = queryVectors.getCentroidDistance(ord); + float lower = queryVectors.getLower(ord); + float width = queryVectors.getWidth(ord); + float normVmC = 0f; + float vDotC = 0f; + if (similarityFunction != EUCLIDEAN) { + normVmC = queryVectors.getNormVmC(ord); + vDotC = queryVectors.getVDotC(ord); + } + BinaryQueryVector binaryQueryVector = new BinaryQueryVector( + vector, + new BinaryQuantizer.QueryFactors(quantizedSum, distanceToCentroid, lower, width, normVmC, vDotC) + ); + return new BinarizedRandomVectorScorer(binaryQueryVector, targetVectors, similarityFunction); + } + + @Override + public RandomVectorScorerSupplier copy() throws IOException { + return new BinarizedRandomVectorScorerSupplier(queryVectors.copy(), targetVectors.copy(), similarityFunction); + } + } + + /** A binarized query representing its quantized form along with factors */ + record BinaryQueryVector(byte[] vector, BinaryQuantizer.QueryFactors factors) {} + + /** Vector scorer over binarized vector values */ + static class BinarizedRandomVectorScorer extends RandomVectorScorer.AbstractRandomVectorScorer { + private final BinaryQueryVector queryVector; + private final BinarizedByteVectorValues targetVectors; + private final VectorSimilarityFunction similarityFunction; + + private final float sqrtDimensions; + private final float maxX1; + + BinarizedRandomVectorScorer( + BinaryQueryVector queryVectors, + BinarizedByteVectorValues targetVectors, + VectorSimilarityFunction similarityFunction + ) { + super(targetVectors); + this.queryVector = queryVectors; + this.targetVectors = targetVectors; + this.similarityFunction = similarityFunction; + // FIXME: precompute this once? + this.sqrtDimensions = targetVectors.sqrtDimensions(); + this.maxX1 = targetVectors.maxX1(); + } + + @Override + public float score(int targetOrd) throws IOException { + byte[] quantizedQuery = queryVector.vector(); + int quantizedSum = queryVector.factors().quantizedSum(); + float lower = queryVector.factors().lower(); + float width = queryVector.factors().width(); + float distanceToCentroid = queryVector.factors().distToC(); + if (similarityFunction == EUCLIDEAN) { + return euclideanScore(targetOrd, sqrtDimensions, quantizedQuery, distanceToCentroid, lower, quantizedSum, width); + } + + float vmC = queryVector.factors().normVmC(); + float vDotC = queryVector.factors().vDotC(); + float cDotC = targetVectors.getCentroidDP(); + byte[] binaryCode = targetVectors.vectorValue(targetOrd); + float ooq = targetVectors.getOOQ(targetOrd); + float normOC = targetVectors.getNormOC(targetOrd); + float oDotC = targetVectors.getODotC(targetOrd); + + float qcDist = ESVectorUtil.ipByteBinByte(quantizedQuery, binaryCode); + + // FIXME: pre-compute these only once for each target vector + // ... pull this out or use a similar cache mechanism as do in score + float xbSum = (float) BQVectorUtils.popcount(binaryCode); + final float dist; + // If ||o-c|| == 0, so, it's ok to throw the rest of the equation away + // and simply use `oDotC + vDotC - cDotC` as centroid == doc vector + if (normOC == 0 || ooq == 0) { + dist = oDotC + vDotC - cDotC; + } else { + // If ||o-c|| != 0, we should assume that `ooq` is finite + assert Float.isFinite(ooq); + float estimatedDot = (2 * width / sqrtDimensions * qcDist + 2 * lower / sqrtDimensions * xbSum - width / sqrtDimensions + * quantizedSum - sqrtDimensions * lower) / ooq; + dist = vmC * normOC * estimatedDot + oDotC + vDotC - cDotC; + } + assert Float.isFinite(dist); + + float ooqSqr = (float) Math.pow(ooq, 2); + float errorBound = (float) (vmC * normOC * (maxX1 * Math.sqrt((1 - ooqSqr) / ooqSqr))); + float score = Float.isFinite(errorBound) ? dist - errorBound : dist; + if (similarityFunction == MAXIMUM_INNER_PRODUCT) { + return VectorUtil.scaleMaxInnerProductScore(score); + } + return Math.max((1f + score) / 2f, 0); + } + + private float euclideanScore( + int targetOrd, + float sqrtDimensions, + byte[] quantizedQuery, + float distanceToCentroid, + float lower, + int quantizedSum, + float width + ) throws IOException { + byte[] binaryCode = targetVectors.vectorValue(targetOrd); + + // FIXME: pre-compute these only once for each target vector + // .. not sure how to enumerate the target ordinals but that's what we did in PoC + float targetDistToC = targetVectors.getCentroidDistance(targetOrd); + float x0 = targetVectors.getVectorMagnitude(targetOrd); + float sqrX = targetDistToC * targetDistToC; + double xX0 = targetDistToC / x0; + + // TODO maybe store? + float xbSum = (float) BQVectorUtils.popcount(binaryCode); + float factorPPC = (float) (-2.0 / sqrtDimensions * xX0 * (xbSum * 2.0 - targetVectors.dimension())); + float factorIP = (float) (-2.0 / sqrtDimensions * xX0); + + long qcDist = ESVectorUtil.ipByteBinByte(quantizedQuery, binaryCode); + float score = sqrX + distanceToCentroid + factorPPC * lower + (qcDist * 2 - quantizedSum) * factorIP * width; + float projectionDist = (float) Math.sqrt(xX0 * xX0 - targetDistToC * targetDistToC); + float error = 2.0f * maxX1 * projectionDist; + float y = (float) Math.sqrt(distanceToCentroid); + float errorBound = y * error; + if (Float.isFinite(errorBound)) { + score = score + errorBound; + } + return Math.max(1 / (1f + score), 0); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatVectorsScorerTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatVectorsScorerTests.java index a75b9bc6064d1..ffe007be9799d 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatVectorsScorerTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatVectorsScorerTests.java @@ -59,7 +59,7 @@ public void testScore() throws IOException { short quantizedSum = (short) random().nextInt(0, 4097); float normVmC = random().nextFloat(-1000f, 1000f); float vDotC = random().nextFloat(-1000f, 1000f); - ES816BinaryFlatVectorsScorer.BinaryQueryVector queryVector = new ES816BinaryFlatVectorsScorer.BinaryQueryVector( + ES816BinaryFlatRWVectorsScorer.BinaryQueryVector queryVector = new ES816BinaryFlatRWVectorsScorer.BinaryQueryVector( vector, new BinaryQuantizer.QueryFactors(quantizedSum, distanceToCentroid, vl, width, normVmC, vDotC) ); @@ -134,7 +134,7 @@ public int dimension() { } }; - ES816BinaryFlatVectorsScorer.BinarizedRandomVectorScorer scorer = new ES816BinaryFlatVectorsScorer.BinarizedRandomVectorScorer( + ES816BinaryFlatRWVectorsScorer.BinarizedRandomVectorScorer scorer = new ES816BinaryFlatRWVectorsScorer.BinarizedRandomVectorScorer( queryVector, targetVectors, similarityFunction @@ -217,7 +217,7 @@ public void testScoreEuclidean() throws IOException { float vl = -57.883f; float width = 9.972266f; short quantizedSum = 795; - ES816BinaryFlatVectorsScorer.BinaryQueryVector queryVector = new ES816BinaryFlatVectorsScorer.BinaryQueryVector( + ES816BinaryFlatRWVectorsScorer.BinaryQueryVector queryVector = new ES816BinaryFlatRWVectorsScorer.BinaryQueryVector( vector, new BinaryQuantizer.QueryFactors(quantizedSum, distanceToCentroid, vl, width, 0f, 0f) ); @@ -420,7 +420,7 @@ public int dimension() { VectorSimilarityFunction similarityFunction = VectorSimilarityFunction.EUCLIDEAN; - ES816BinaryFlatVectorsScorer.BinarizedRandomVectorScorer scorer = new ES816BinaryFlatVectorsScorer.BinarizedRandomVectorScorer( + ES816BinaryFlatRWVectorsScorer.BinarizedRandomVectorScorer scorer = new ES816BinaryFlatRWVectorsScorer.BinarizedRandomVectorScorer( queryVector, targetVectors, similarityFunction @@ -824,7 +824,7 @@ public void testScoreMIP() throws IOException { float normVmC = 9.766797f; float vDotC = 133.56123f; float cDotC = 132.20227f; - ES816BinaryFlatVectorsScorer.BinaryQueryVector queryVector = new ES816BinaryFlatVectorsScorer.BinaryQueryVector( + ES816BinaryFlatRWVectorsScorer.BinaryQueryVector queryVector = new ES816BinaryFlatRWVectorsScorer.BinaryQueryVector( vector, new BinaryQuantizer.QueryFactors(quantizedSum, distanceToCentroid, vl, width, normVmC, vDotC) ); @@ -1768,7 +1768,7 @@ public int dimension() { VectorSimilarityFunction similarityFunction = VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT; - ES816BinaryFlatVectorsScorer.BinarizedRandomVectorScorer scorer = new ES816BinaryFlatVectorsScorer.BinarizedRandomVectorScorer( + ES816BinaryFlatRWVectorsScorer.BinarizedRandomVectorScorer scorer = new ES816BinaryFlatRWVectorsScorer.BinarizedRandomVectorScorer( queryVector, targetVectors, similarityFunction diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedRWVectorsFormat.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedRWVectorsFormat.java new file mode 100644 index 0000000000000..c54903a94b54f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedRWVectorsFormat.java @@ -0,0 +1,52 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors.es816; + +import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; +import org.apache.lucene.index.SegmentWriteState; + +import java.io.IOException; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + */ +public class ES816BinaryQuantizedRWVectorsFormat extends ES816BinaryQuantizedVectorsFormat { + + private static final FlatVectorsFormat rawVectorFormat = new Lucene99FlatVectorsFormat( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() + ); + + private static final ES816BinaryFlatRWVectorsScorer scorer = new ES816BinaryFlatRWVectorsScorer( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() + ); + + /** Creates a new instance with the default number of vectors per cluster. */ + public ES816BinaryQuantizedRWVectorsFormat() { + super(); + } + + @Override + public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new ES816BinaryQuantizedVectorsWriter(scorer, rawVectorFormat.fieldsWriter(state), state); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java index 681f615653d40..48ba566353f5d 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java @@ -63,7 +63,7 @@ protected Codec getCodec() { return new Lucene100Codec() { @Override public KnnVectorsFormat getKnnVectorsFormatForField(String field) { - return new ES816BinaryQuantizedVectorsFormat(); + return new ES816BinaryQuantizedRWVectorsFormat(); } }; } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java similarity index 99% rename from server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java rename to server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java index 31ae977e81118..4d97235c5fae5 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java @@ -77,7 +77,7 @@ class ES816BinaryQuantizedVectorsWriter extends FlatVectorsWriter { private final List fields = new ArrayList<>(); private final IndexOutput meta, binarizedVectorData; private final FlatVectorsWriter rawVectorDelegate; - private final ES816BinaryFlatVectorsScorer vectorsScorer; + private final ES816BinaryFlatRWVectorsScorer vectorsScorer; private boolean finished; /** @@ -86,7 +86,7 @@ class ES816BinaryQuantizedVectorsWriter extends FlatVectorsWriter { * @param vectorsScorer the scorer to use for scoring vectors */ protected ES816BinaryQuantizedVectorsWriter( - ES816BinaryFlatVectorsScorer vectorsScorer, + ES816BinaryFlatRWVectorsScorer vectorsScorer, FlatVectorsWriter rawVectorDelegate, SegmentWriteState state ) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedRWVectorsFormat.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedRWVectorsFormat.java new file mode 100644 index 0000000000000..e9bace72b591c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedRWVectorsFormat.java @@ -0,0 +1,55 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors.es816; + +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; +import org.apache.lucene.index.SegmentWriteState; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; + +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; + +class ES816HnswBinaryQuantizedRWVectorsFormat extends ES816HnswBinaryQuantizedVectorsFormat { + + private static final FlatVectorsFormat flatVectorsFormat = new ES816BinaryQuantizedRWVectorsFormat(); + + /** Constructs a format using default graph construction parameters */ + ES816HnswBinaryQuantizedRWVectorsFormat() { + this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH); + } + + ES816HnswBinaryQuantizedRWVectorsFormat(int maxConn, int beamWidth) { + this(maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, null); + } + + ES816HnswBinaryQuantizedRWVectorsFormat(int maxConn, int beamWidth, int numMergeWorkers, ExecutorService mergeExec) { + super(maxConn, beamWidth, numMergeWorkers, mergeExec); + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new Lucene99HnswVectorsWriter(state, maxConn, beamWidth, flatVectorsFormat.fieldsWriter(state), 1, null); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java index a25fa2836ee34..03aa847f3a5d4 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java @@ -59,7 +59,7 @@ protected Codec getCodec() { return new Lucene100Codec() { @Override public KnnVectorsFormat getKnnVectorsFormatForField(String field) { - return new ES816HnswBinaryQuantizedVectorsFormat(); + return new ES816HnswBinaryQuantizedRWVectorsFormat(); } }; } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java new file mode 100644 index 0000000000000..397cc472592b6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java @@ -0,0 +1,181 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors.es818; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene100.Lucene100Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.codec.vectors.BQVectorUtils; + +import java.io.IOException; +import java.util.Locale; + +import static java.lang.String.format; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; + +public class ES818BinaryQuantizedVectorsFormatTests extends BaseKnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + @Override + protected Codec getCodec() { + return new Lucene100Codec() { + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return new ES818BinaryQuantizedVectorsFormat(); + } + }; + } + + public void testSearch() throws Exception { + String fieldName = "field"; + int numVectors = random().nextInt(99, 500); + int dims = random().nextInt(4, 65); + float[] vector = randomVector(dims); + VectorSimilarityFunction similarityFunction = randomSimilarity(); + KnnFloatVectorField knnField = new KnnFloatVectorField(fieldName, vector, similarityFunction); + IndexWriterConfig iwc = newIndexWriterConfig(); + try (Directory dir = newDirectory()) { + try (IndexWriter w = new IndexWriter(dir, iwc)) { + for (int i = 0; i < numVectors; i++) { + Document doc = new Document(); + knnField.setVectorValue(randomVector(dims)); + doc.add(knnField); + w.addDocument(doc); + } + w.commit(); + + try (IndexReader reader = DirectoryReader.open(w)) { + IndexSearcher searcher = new IndexSearcher(reader); + final int k = random().nextInt(5, 50); + float[] queryVector = randomVector(dims); + Query q = new KnnFloatVectorQuery(fieldName, queryVector, k); + TopDocs collectedDocs = searcher.search(q, k); + assertEquals(k, collectedDocs.totalHits.value()); + assertEquals(TotalHits.Relation.EQUAL_TO, collectedDocs.totalHits.relation()); + } + } + } + } + + public void testToString() { + FilterCodec customCodec = new FilterCodec("foo", Codec.getDefault()) { + @Override + public KnnVectorsFormat knnVectorsFormat() { + return new ES818BinaryQuantizedVectorsFormat(); + } + }; + String expectedPattern = "ES818BinaryQuantizedVectorsFormat(" + + "name=ES818BinaryQuantizedVectorsFormat, " + + "flatVectorScorer=ES818BinaryFlatVectorsScorer(nonQuantizedDelegate=%s()))"; + var defaultScorer = format(Locale.ROOT, expectedPattern, "DefaultFlatVectorScorer"); + var memSegScorer = format(Locale.ROOT, expectedPattern, "Lucene99MemorySegmentFlatVectorsScorer"); + assertThat(customCodec.knnVectorsFormat().toString(), is(oneOf(defaultScorer, memSegScorer))); + } + + @Override + public void testRandomWithUpdatesAndGraph() { + // graph not supported + } + + @Override + public void testSearchWithVisitedLimit() { + // visited limit is not respected, as it is brute force search + } + + public void testQuantizedVectorsWriteAndRead() throws IOException { + String fieldName = "field"; + int numVectors = random().nextInt(99, 500); + int dims = random().nextInt(4, 65); + + float[] vector = randomVector(dims); + VectorSimilarityFunction similarityFunction = randomSimilarity(); + KnnFloatVectorField knnField = new KnnFloatVectorField(fieldName, vector, similarityFunction); + try (Directory dir = newDirectory()) { + try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + for (int i = 0; i < numVectors; i++) { + Document doc = new Document(); + knnField.setVectorValue(randomVector(dims)); + doc.add(knnField); + w.addDocument(doc); + if (i % 101 == 0) { + w.commit(); + } + } + w.commit(); + w.forceMerge(1); + + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName); + assertEquals(vectorValues.size(), numVectors); + BinarizedByteVectorValues qvectorValues = ((ES818BinaryQuantizedVectorsReader.BinarizedVectorValues) vectorValues) + .getQuantizedVectorValues(); + float[] centroid = qvectorValues.getCentroid(); + assertEquals(centroid.length, dims); + + OptimizedScalarQuantizer quantizer = new OptimizedScalarQuantizer(similarityFunction); + byte[] quantizedVector = new byte[dims]; + byte[] expectedVector = new byte[BQVectorUtils.discretize(dims, 64) / 8]; + if (similarityFunction == VectorSimilarityFunction.COSINE) { + vectorValues = new ES818BinaryQuantizedVectorsWriter.NormalizedFloatVectorValues(vectorValues); + } + KnnVectorValues.DocIndexIterator docIndexIterator = vectorValues.iterator(); + + while (docIndexIterator.nextDoc() != NO_MORE_DOCS) { + OptimizedScalarQuantizer.QuantizationResult corrections = quantizer.scalarQuantize( + vectorValues.vectorValue(docIndexIterator.index()), + quantizedVector, + (byte) 1, + centroid + ); + BQVectorUtils.packAsBinary(quantizedVector, expectedVector); + assertArrayEquals(expectedVector, qvectorValues.vectorValue(docIndexIterator.index())); + assertEquals(corrections, qvectorValues.getCorrectiveTerms(docIndexIterator.index())); + } + } + } + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java new file mode 100644 index 0000000000000..b6ae3199bb896 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java @@ -0,0 +1,132 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors.es818; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene100.Lucene100Codec; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.apache.lucene.util.SameThreadExecutorService; +import org.elasticsearch.common.logging.LogConfigurator; + +import java.util.Arrays; +import java.util.Locale; + +import static java.lang.String.format; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; + +public class ES818HnswBinaryQuantizedVectorsFormatTests extends BaseKnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + @Override + protected Codec getCodec() { + return new Lucene100Codec() { + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return new ES818HnswBinaryQuantizedVectorsFormat(); + } + }; + } + + public void testToString() { + FilterCodec customCodec = new FilterCodec("foo", Codec.getDefault()) { + @Override + public KnnVectorsFormat knnVectorsFormat() { + return new ES818HnswBinaryQuantizedVectorsFormat(10, 20, 1, null); + } + }; + String expectedPattern = + "ES818HnswBinaryQuantizedVectorsFormat(name=ES818HnswBinaryQuantizedVectorsFormat, maxConn=10, beamWidth=20," + + " flatVectorFormat=ES818BinaryQuantizedVectorsFormat(name=ES818BinaryQuantizedVectorsFormat," + + " flatVectorScorer=ES818BinaryFlatVectorsScorer(nonQuantizedDelegate=%s())))"; + + var defaultScorer = format(Locale.ROOT, expectedPattern, "DefaultFlatVectorScorer"); + var memSegScorer = format(Locale.ROOT, expectedPattern, "Lucene99MemorySegmentFlatVectorsScorer"); + assertThat(customCodec.knnVectorsFormat().toString(), is(oneOf(defaultScorer, memSegScorer))); + } + + public void testSingleVectorCase() throws Exception { + float[] vector = randomVector(random().nextInt(12, 500)); + for (VectorSimilarityFunction similarityFunction : VectorSimilarityFunction.values()) { + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc = new Document(); + doc.add(new KnnFloatVectorField("f", vector, similarityFunction)); + w.addDocument(doc); + w.commit(); + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues vectorValues = r.getFloatVectorValues("f"); + KnnVectorValues.DocIndexIterator docIndexIterator = vectorValues.iterator(); + assert (vectorValues.size() == 1); + while (docIndexIterator.nextDoc() != NO_MORE_DOCS) { + assertArrayEquals(vector, vectorValues.vectorValue(docIndexIterator.index()), 0.00001f); + } + float[] randomVector = randomVector(vector.length); + float trueScore = similarityFunction.compare(vector, randomVector); + TopDocs td = r.searchNearestVectors("f", randomVector, 1, null, Integer.MAX_VALUE); + assertEquals(1, td.totalHits.value()); + assertTrue(td.scoreDocs[0].score >= 0); + // When it's the only vector in a segment, the score should be very close to the true score + assertEquals(trueScore, td.scoreDocs[0].score, 0.0001f); + } + } + } + } + + public void testLimits() { + expectThrows(IllegalArgumentException.class, () -> new ES818HnswBinaryQuantizedVectorsFormat(-1, 20)); + expectThrows(IllegalArgumentException.class, () -> new ES818HnswBinaryQuantizedVectorsFormat(0, 20)); + expectThrows(IllegalArgumentException.class, () -> new ES818HnswBinaryQuantizedVectorsFormat(20, 0)); + expectThrows(IllegalArgumentException.class, () -> new ES818HnswBinaryQuantizedVectorsFormat(20, -1)); + expectThrows(IllegalArgumentException.class, () -> new ES818HnswBinaryQuantizedVectorsFormat(512 + 1, 20)); + expectThrows(IllegalArgumentException.class, () -> new ES818HnswBinaryQuantizedVectorsFormat(20, 3201)); + expectThrows( + IllegalArgumentException.class, + () -> new ES818HnswBinaryQuantizedVectorsFormat(20, 100, 1, new SameThreadExecutorService()) + ); + } + + // Ensures that all expected vector similarity functions are translatable in the format. + public void testVectorSimilarityFuncs() { + // This does not necessarily have to be all similarity functions, but + // differences should be considered carefully. + var expectedValues = Arrays.stream(VectorSimilarityFunction.values()).toList(); + assertEquals(Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS, expectedValues); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/OptimizedScalarQuantizerTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/OptimizedScalarQuantizerTests.java new file mode 100644 index 0000000000000..e3e2d6caafe0e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/OptimizedScalarQuantizerTests.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es818; + +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.util.VectorUtil; +import org.elasticsearch.test.ESTestCase; + +import static org.elasticsearch.index.codec.vectors.es818.OptimizedScalarQuantizer.MINIMUM_MSE_GRID; + +public class OptimizedScalarQuantizerTests extends ESTestCase { + + static final byte[] ALL_BITS = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }; + + public void testAbusiveEdgeCases() { + // large zero array + for (VectorSimilarityFunction vectorSimilarityFunction : VectorSimilarityFunction.values()) { + if (vectorSimilarityFunction == VectorSimilarityFunction.COSINE) { + continue; + } + float[] vector = new float[4096]; + float[] centroid = new float[4096]; + OptimizedScalarQuantizer osq = new OptimizedScalarQuantizer(vectorSimilarityFunction); + byte[][] destinations = new byte[MINIMUM_MSE_GRID.length][4096]; + OptimizedScalarQuantizer.QuantizationResult[] results = osq.multiScalarQuantize(vector, destinations, ALL_BITS, centroid); + assertEquals(MINIMUM_MSE_GRID.length, results.length); + assertValidResults(results); + for (byte[] destination : destinations) { + assertArrayEquals(new byte[4096], destination); + } + byte[] destination = new byte[4096]; + for (byte bit : ALL_BITS) { + OptimizedScalarQuantizer.QuantizationResult result = osq.scalarQuantize(vector, destination, bit, centroid); + assertValidResults(result); + assertArrayEquals(new byte[4096], destination); + } + } + + // single value array + for (VectorSimilarityFunction vectorSimilarityFunction : VectorSimilarityFunction.values()) { + float[] vector = new float[] { randomFloat() }; + float[] centroid = new float[] { randomFloat() }; + if (vectorSimilarityFunction == VectorSimilarityFunction.COSINE) { + VectorUtil.l2normalize(vector); + VectorUtil.l2normalize(centroid); + } + OptimizedScalarQuantizer osq = new OptimizedScalarQuantizer(vectorSimilarityFunction); + byte[][] destinations = new byte[MINIMUM_MSE_GRID.length][1]; + OptimizedScalarQuantizer.QuantizationResult[] results = osq.multiScalarQuantize(vector, destinations, ALL_BITS, centroid); + assertEquals(MINIMUM_MSE_GRID.length, results.length); + assertValidResults(results); + for (int i = 0; i < ALL_BITS.length; i++) { + assertValidQuantizedRange(destinations[i], ALL_BITS[i]); + } + for (byte bit : ALL_BITS) { + vector = new float[] { randomFloat() }; + centroid = new float[] { randomFloat() }; + if (vectorSimilarityFunction == VectorSimilarityFunction.COSINE) { + VectorUtil.l2normalize(vector); + VectorUtil.l2normalize(centroid); + } + byte[] destination = new byte[1]; + OptimizedScalarQuantizer.QuantizationResult result = osq.scalarQuantize(vector, destination, bit, centroid); + assertValidResults(result); + assertValidQuantizedRange(destination, bit); + } + } + + } + + public void testMathematicalConsistency() { + int dims = randomIntBetween(1, 4096); + float[] vector = new float[dims]; + for (int i = 0; i < dims; ++i) { + vector[i] = randomFloat(); + } + float[] centroid = new float[dims]; + for (int i = 0; i < dims; ++i) { + centroid[i] = randomFloat(); + } + float[] copy = new float[dims]; + for (VectorSimilarityFunction vectorSimilarityFunction : VectorSimilarityFunction.values()) { + // copy the vector to avoid modifying it + System.arraycopy(vector, 0, copy, 0, dims); + if (vectorSimilarityFunction == VectorSimilarityFunction.COSINE) { + VectorUtil.l2normalize(copy); + VectorUtil.l2normalize(centroid); + } + OptimizedScalarQuantizer osq = new OptimizedScalarQuantizer(vectorSimilarityFunction); + byte[][] destinations = new byte[MINIMUM_MSE_GRID.length][dims]; + OptimizedScalarQuantizer.QuantizationResult[] results = osq.multiScalarQuantize(copy, destinations, ALL_BITS, centroid); + assertEquals(MINIMUM_MSE_GRID.length, results.length); + assertValidResults(results); + for (int i = 0; i < ALL_BITS.length; i++) { + assertValidQuantizedRange(destinations[i], ALL_BITS[i]); + } + for (byte bit : ALL_BITS) { + byte[] destination = new byte[dims]; + System.arraycopy(vector, 0, copy, 0, dims); + if (vectorSimilarityFunction == VectorSimilarityFunction.COSINE) { + VectorUtil.l2normalize(copy); + VectorUtil.l2normalize(centroid); + } + OptimizedScalarQuantizer.QuantizationResult result = osq.scalarQuantize(copy, destination, bit, centroid); + assertValidResults(result); + assertValidQuantizedRange(destination, bit); + } + } + } + + static void assertValidQuantizedRange(byte[] quantized, byte bits) { + for (byte b : quantized) { + if (bits < 8) { + assertTrue(b >= 0); + } + assertTrue(b < 1 << bits); + } + } + + static void assertValidResults(OptimizedScalarQuantizer.QuantizationResult... results) { + for (OptimizedScalarQuantizer.QuantizationResult result : results) { + assertTrue(Float.isFinite(result.lowerInterval())); + assertTrue(Float.isFinite(result.upperInterval())); + assertTrue(result.lowerInterval() <= result.upperInterval()); + assertTrue(Float.isFinite(result.additionalCorrection())); + assertTrue(result.quantizedComponentSum() >= 0); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index de084cd4582e2..c043b9ffb381a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -1970,13 +1970,13 @@ public void testKnnBBQHNSWVectorsFormat() throws IOException { assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } - String expectedString = "ES816HnswBinaryQuantizedVectorsFormat(name=ES816HnswBinaryQuantizedVectorsFormat, maxConn=" + String expectedString = "ES818HnswBinaryQuantizedVectorsFormat(name=ES818HnswBinaryQuantizedVectorsFormat, maxConn=" + m + ", beamWidth=" + efConstruction - + ", flatVectorFormat=ES816BinaryQuantizedVectorsFormat(" - + "name=ES816BinaryQuantizedVectorsFormat, " - + "flatVectorScorer=ES816BinaryFlatVectorsScorer(nonQuantizedDelegate=DefaultFlatVectorScorer())))"; + + ", flatVectorFormat=ES818BinaryQuantizedVectorsFormat(" + + "name=ES818BinaryQuantizedVectorsFormat, " + + "flatVectorScorer=ES818BinaryFlatVectorsScorer(nonQuantizedDelegate=DefaultFlatVectorScorer())))"; assertEquals(expectedString, knnVectorsFormat.toString()); } From 4b868b0e11f4a59d608523c57d1a62b870ac8e0e Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:57:40 +0100 Subject: [PATCH 134/204] Fix enrich cache size setting name (#117575) The enrich cache size setting accidentally got renamed from `enrich.cache_size` to `enrich.cache.size` in #111412. This commit updates the enrich plugin to accept both names and deprecates the wrong name. --- docs/changelog/117575.yaml | 5 ++ .../xpack/enrich/EnrichPlugin.java | 59 +++++++++++++++++- .../xpack/enrich/EnrichPluginTests.java | 62 +++++++++++++++++++ 3 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/117575.yaml create mode 100644 x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPluginTests.java diff --git a/docs/changelog/117575.yaml b/docs/changelog/117575.yaml new file mode 100644 index 0000000000000..781444ae97be5 --- /dev/null +++ b/docs/changelog/117575.yaml @@ -0,0 +1,5 @@ +pr: 117575 +summary: Fix enrich cache size setting name +area: Ingest Node +type: bug +issues: [] diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java index 1a68ada60b6f1..d46639d700420 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java @@ -14,6 +14,8 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; @@ -23,6 +25,7 @@ import org.elasticsearch.common.unit.MemorySizeValue; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.ingest.Processor; @@ -74,6 +77,8 @@ public class EnrichPlugin extends Plugin implements SystemIndexPlugin, IngestPlugin { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(EnrichPlugin.class); + static final Setting ENRICH_FETCH_SIZE_SETTING = Setting.intSetting( "enrich.fetch_size", 10000, @@ -126,9 +131,9 @@ public class EnrichPlugin extends Plugin implements SystemIndexPlugin, IngestPlu return String.valueOf(maxConcurrentRequests * maxLookupsPerRequest); }, val -> Setting.parseInt(val, 1, Integer.MAX_VALUE, QUEUE_CAPACITY_SETTING_NAME), Setting.Property.NodeScope); - public static final String CACHE_SIZE_SETTING_NAME = "enrich.cache.size"; + public static final String CACHE_SIZE_SETTING_NAME = "enrich.cache_size"; public static final Setting CACHE_SIZE = new Setting<>( - "enrich.cache.size", + CACHE_SIZE_SETTING_NAME, (String) null, (String s) -> FlatNumberOrByteSizeValue.parse( s, @@ -138,16 +143,59 @@ public class EnrichPlugin extends Plugin implements SystemIndexPlugin, IngestPlu Setting.Property.NodeScope ); + /** + * This setting solely exists because the original setting was accidentally renamed in + * https://github.com/elastic/elasticsearch/pull/111412. + */ + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) + public static final String CACHE_SIZE_SETTING_BWC_NAME = "enrich.cache.size"; + public static final Setting CACHE_SIZE_BWC = new Setting<>( + CACHE_SIZE_SETTING_BWC_NAME, + (String) null, + (String s) -> FlatNumberOrByteSizeValue.parse( + s, + CACHE_SIZE_SETTING_BWC_NAME, + new FlatNumberOrByteSizeValue(ByteSizeValue.ofBytes((long) (0.01 * JvmInfo.jvmInfo().getConfiguredMaxHeapSize()))) + ), + Setting.Property.NodeScope, + Setting.Property.Deprecated + ); + private final Settings settings; private final EnrichCache enrichCache; + private final long maxCacheSize; public EnrichPlugin(final Settings settings) { this.settings = settings; - FlatNumberOrByteSizeValue maxSize = CACHE_SIZE.get(settings); + FlatNumberOrByteSizeValue maxSize; + if (settings.hasValue(CACHE_SIZE_SETTING_BWC_NAME)) { + if (settings.hasValue(CACHE_SIZE_SETTING_NAME)) { + throw new IllegalArgumentException( + Strings.format( + "Both [{}] and [{}] are set, please use [{}]", + CACHE_SIZE_SETTING_NAME, + CACHE_SIZE_SETTING_BWC_NAME, + CACHE_SIZE_SETTING_NAME + ) + ); + } + deprecationLogger.warn( + DeprecationCategory.SETTINGS, + "enrich_cache_size_name", + "The [{}] setting is deprecated and will be removed in a future version. Please use [{}] instead.", + CACHE_SIZE_SETTING_BWC_NAME, + CACHE_SIZE_SETTING_NAME + ); + maxSize = CACHE_SIZE_BWC.get(settings); + } else { + maxSize = CACHE_SIZE.get(settings); + } if (maxSize.byteSizeValue() != null) { this.enrichCache = new EnrichCache(maxSize.byteSizeValue()); + this.maxCacheSize = maxSize.byteSizeValue().getBytes(); } else { this.enrichCache = new EnrichCache(maxSize.flatNumber()); + this.maxCacheSize = maxSize.flatNumber(); } } @@ -286,6 +334,11 @@ public String getFeatureDescription() { return "Manages data related to Enrich policies"; } + // Visible for testing + long getMaxCacheSize() { + return maxCacheSize; + } + /** * A class that specifies either a flat (unit-less) number or a byte size value. */ diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPluginTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPluginTests.java new file mode 100644 index 0000000000000..07de0e0967448 --- /dev/null +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPluginTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.enrich; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; + +public class EnrichPluginTests extends ESTestCase { + + public void testConstructWithByteSize() { + final var size = randomNonNegativeInt(); + Settings settings = Settings.builder().put(EnrichPlugin.CACHE_SIZE_SETTING_NAME, size + "b").build(); + EnrichPlugin plugin = new EnrichPlugin(settings); + assertEquals(size, plugin.getMaxCacheSize()); + } + + public void testConstructWithFlatNumber() { + final var size = randomNonNegativeInt(); + Settings settings = Settings.builder().put(EnrichPlugin.CACHE_SIZE_SETTING_NAME, size).build(); + EnrichPlugin plugin = new EnrichPlugin(settings); + assertEquals(size, plugin.getMaxCacheSize()); + } + + public void testConstructWithByteSizeBwc() { + final var size = randomNonNegativeInt(); + Settings settings = Settings.builder().put(EnrichPlugin.CACHE_SIZE_SETTING_BWC_NAME, size + "b").build(); + EnrichPlugin plugin = new EnrichPlugin(settings); + assertEquals(size, plugin.getMaxCacheSize()); + } + + public void testConstructWithFlatNumberBwc() { + final var size = randomNonNegativeInt(); + Settings settings = Settings.builder().put(EnrichPlugin.CACHE_SIZE_SETTING_BWC_NAME, size).build(); + EnrichPlugin plugin = new EnrichPlugin(settings); + assertEquals(size, plugin.getMaxCacheSize()); + } + + public void testConstructWithBothSettings() { + Settings settings = Settings.builder() + .put(EnrichPlugin.CACHE_SIZE_SETTING_NAME, randomNonNegativeInt()) + .put(EnrichPlugin.CACHE_SIZE_SETTING_BWC_NAME, randomNonNegativeInt()) + .build(); + assertThrows(IllegalArgumentException.class, () -> new EnrichPlugin(settings)); + } + + @Override + protected List filteredWarnings() { + final var warnings = super.filteredWarnings(); + warnings.add("[enrich.cache.size] setting was deprecated in Elasticsearch and will be removed in a future release."); + warnings.add( + "The [enrich.cache.size] setting is deprecated and will be removed in a future version. Please use [enrich.cache_size] instead." + ); + return warnings; + } +} From dcae87da47f258f041e164b4d5620b1adf7be090 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Mon, 9 Dec 2024 12:48:29 -0500 Subject: [PATCH 135/204] [ML] Fixing failing inference api streaming tests (#118284) * Fixing tests with alphanumeric strings * Removing prints --- muted-tests.yml | 6 ------ .../org/elasticsearch/test/ESTestCase.java | 20 +++++++++++++++++++ .../test/cluster/FeatureFlag.java | 3 ++- .../inference/InferenceBaseRestTest.java | 2 ++ .../xpack/inference/InferenceCrudIT.java | 4 ++-- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index f922a1a27b629..d356dd2f791d1 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -186,9 +186,6 @@ tests: - class: "org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT" method: "test {scoring.*}" issue: https://github.com/elastic/elasticsearch/issues/117641 -- class: org.elasticsearch.xpack.inference.InferenceCrudIT - method: testSupportedStream - issue: https://github.com/elastic/elasticsearch/issues/117745 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT method: test {scoring.QstrWithFieldAndScoringSortedEval} issue: https://github.com/elastic/elasticsearch/issues/117751 @@ -272,9 +269,6 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveTests method: test51AutoConfigurationWithPasswordProtectedKeystore issue: https://github.com/elastic/elasticsearch/issues/118212 -- class: org.elasticsearch.xpack.inference.InferenceCrudIT - method: testUnifiedCompletionInference - issue: https://github.com/elastic/elasticsearch/issues/118210 - class: org.elasticsearch.ingest.common.IngestCommonClientYamlTestSuiteIT issue: https://github.com/elastic/elasticsearch/issues/118215 - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index a71f61740e17b..e869fc0836ba6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -277,6 +277,11 @@ public static void resetPortCounter() { private static final SetOnce WARN_SECURE_RANDOM_FIPS_NOT_DETERMINISTIC = new SetOnce<>(); + private static final String LOWER_ALPHA_CHARACTERS = "abcdefghijklmnopqrstuvwxyz"; + private static final String UPPER_ALPHA_CHARACTERS = LOWER_ALPHA_CHARACTERS.toUpperCase(Locale.ROOT); + private static final String DIGIT_CHARACTERS = "0123456789"; + private static final String ALPHANUMERIC_CHARACTERS = LOWER_ALPHA_CHARACTERS + UPPER_ALPHA_CHARACTERS + DIGIT_CHARACTERS; + static { Random random = initTestSeed(); TEST_WORKER_VM_ID = System.getProperty(TEST_WORKER_SYS_PROPERTY, DEFAULT_TEST_WORKER_ID); @@ -1200,6 +1205,21 @@ public static String randomAlphaOfLength(int codeUnits) { return RandomizedTest.randomAsciiOfLength(codeUnits); } + /** + * Generate a random string containing only alphanumeric characters. + * @param length the length of the string to generate + * @return the generated string + */ + public static String randomAlphanumericOfLength(int length) { + StringBuilder sb = new StringBuilder(); + Random random = random(); + for (int i = 0; i < length; i++) { + sb.append(ALPHANUMERIC_CHARACTERS.charAt(random.nextInt(ALPHANUMERIC_CHARACTERS.length()))); + } + + return sb.toString(); + } + public static SecureString randomSecureStringOfLength(int codeUnits) { var randomAlpha = randomAlphaOfLength(codeUnits); return new SecureString(randomAlpha.toCharArray()); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index 11787866af0d7..5630c33ad559c 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -18,7 +18,8 @@ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null), - SUB_OBJECTS_AUTO_ENABLED("es.sub_objects_auto_feature_flag_enabled=true", Version.fromString("8.16.0"), null); + SUB_OBJECTS_AUTO_ENABLED("es.sub_objects_auto_feature_flag_enabled=true", Version.fromString("8.16.0"), null), + INFERENCE_UNIFIED_API_ENABLED("es.inference_unified_feature_flag_enabled=true", Version.fromString("8.18.0"), null); public final String systemProperty; public final Version from; diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java index 07ce2fe00642b..5b7394e89bc43 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java @@ -19,6 +19,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; @@ -46,6 +47,7 @@ public class InferenceBaseRestTest extends ESRestTestCase { .setting("xpack.security.enabled", "true") .plugin("inference-service-test") .user("x_pack_rest_user", "x-pack-test-password") + .feature(FeatureFlag.INFERENCE_UNIFIED_API_ENABLED) .build(); @Override diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index 1e19491aeaa60..da1d10db4da8b 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -466,7 +466,7 @@ public void testSupportedStream() throws Exception { assertEquals(modelId, singleModel.get("inference_id")); assertEquals(TaskType.COMPLETION.toString(), singleModel.get("task_type")); - var input = IntStream.range(1, 2 + randomInt(8)).mapToObj(i -> randomUUID()).toList(); + var input = IntStream.range(1, 2 + randomInt(8)).mapToObj(i -> randomAlphanumericOfLength(5)).toList(); try { var events = streamInferOnMockService(modelId, TaskType.COMPLETION, input); @@ -493,7 +493,7 @@ public void testUnifiedCompletionInference() throws Exception { assertEquals(modelId, singleModel.get("inference_id")); assertEquals(TaskType.COMPLETION.toString(), singleModel.get("task_type")); - var input = IntStream.range(1, 2 + randomInt(8)).mapToObj(i -> randomUUID()).toList(); + var input = IntStream.range(1, 2 + randomInt(8)).mapToObj(i -> randomAlphanumericOfLength(5)).toList(); try { var events = unifiedCompletionInferOnMockService(modelId, TaskType.COMPLETION, input); var expectedResponses = expectedResultsIterator(input); From 31678a377df0681c0fca6d45675174df04f0b7e3 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Mon, 9 Dec 2024 13:35:21 -0500 Subject: [PATCH 136/204] Rename multi-dense vector to rank vectors (#118183) renames `multi_dense_vector` field mapper and such to `rank_vectors` to better describe its restricted usage. --- .../org.elasticsearch.script.fields.txt | 10 +- .../org.elasticsearch.script.score.txt | 4 +- .../painless/org.elasticsearch.txt | 2 +- ...x_sim.yml => 141_rank_vectors_max_sim.yml} | 10 +- ...yml => 181_rank_vectors_dv_fields_api.yml} | 10 +- ...i_dense_vector.yml => 30_rank_vectors.yml} | 14 +-- ...a.java => RankVectorsDVLeafFieldData.java} | 18 +-- ...apper.java => RankVectorsFieldMapper.java} | 47 ++++--- ...ta.java => RankVectorsIndexFieldData.java} | 14 +-- ...s.java => RankVectorsScriptDocValues.java} | 24 ++-- .../mapper/vectors/VectorEncoderDecoder.java | 10 -- .../elasticsearch/indices/IndicesModule.java | 6 +- .../action/search/SearchCapabilities.java | 22 ++-- ....java => RankVectorsScoreScriptUtils.java} | 50 ++++---- ...tiDenseVector.java => BitRankVectors.java} | 4 +- ...java => BitRankVectorsDocValuesField.java} | 14 +-- ...iDenseVector.java => ByteRankVectors.java} | 4 +- ...ava => ByteRankVectorsDocValuesField.java} | 22 ++-- ...DenseVector.java => FloatRankVectors.java} | 4 +- ...va => FloatRankVectorsDocValuesField.java} | 22 ++-- ...MultiDenseVector.java => RankVectors.java} | 8 +- ...ld.java => RankVectorsDocValuesField.java} | 20 +-- ....java => RankVectorsFieldMapperTests.java} | 65 +++++----- ...ts.java => RankVectorsFieldTypeTests.java} | 48 ++++---- ...a => RankVectorsScriptDocValuesTests.java} | 116 +++++------------- ... => RankVectorsScoreScriptUtilsTests.java} | 84 ++++++------- ...VectorTests.java => RankVectorsTests.java} | 18 +-- .../aggregations/AggregatorTestCase.java | 4 +- 28 files changed, 295 insertions(+), 379 deletions(-) rename modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/{141_multi_dense_vector_max_sim.yml => 141_rank_vectors_max_sim.yml} (95%) rename modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/{181_multi_dense_vector_dv_fields_api.yml => 181_rank_vectors_dv_fields_api.yml} (94%) rename rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/{30_multi_dense_vector.yml => 30_rank_vectors.yml} (88%) rename server/src/main/java/org/elasticsearch/index/mapper/vectors/{MultiVectorDVLeafFieldData.java => RankVectorsDVLeafFieldData.java} (70%) rename server/src/main/java/org/elasticsearch/index/mapper/vectors/{MultiDenseVectorFieldMapper.java => RankVectorsFieldMapper.java} (90%) rename server/src/main/java/org/elasticsearch/index/mapper/vectors/{MultiVectorIndexFieldData.java => RankVectorsIndexFieldData.java} (87%) rename server/src/main/java/org/elasticsearch/index/mapper/vectors/{MultiDenseVectorScriptDocValues.java => RankVectorsScriptDocValues.java} (70%) rename server/src/main/java/org/elasticsearch/script/{MultiVectorScoreScriptUtils.java => RankVectorsScoreScriptUtils.java} (85%) rename server/src/main/java/org/elasticsearch/script/field/vectors/{BitMultiDenseVector.java => BitRankVectors.java} (94%) rename server/src/main/java/org/elasticsearch/script/field/vectors/{BitMultiDenseVectorDocValuesField.java => BitRankVectorsDocValuesField.java} (64%) rename server/src/main/java/org/elasticsearch/script/field/vectors/{ByteMultiDenseVector.java => ByteRankVectors.java} (95%) rename server/src/main/java/org/elasticsearch/script/field/vectors/{ByteMultiDenseVectorDocValuesField.java => ByteRankVectorsDocValuesField.java} (85%) rename server/src/main/java/org/elasticsearch/script/field/vectors/{FloatMultiDenseVector.java => FloatRankVectors.java} (93%) rename server/src/main/java/org/elasticsearch/script/field/vectors/{FloatMultiDenseVectorDocValuesField.java => FloatRankVectorsDocValuesField.java} (85%) rename server/src/main/java/org/elasticsearch/script/field/vectors/{MultiDenseVector.java => RankVectors.java} (92%) rename server/src/main/java/org/elasticsearch/script/field/vectors/{MultiDenseVectorDocValuesField.java => RankVectorsDocValuesField.java} (68%) rename server/src/test/java/org/elasticsearch/index/mapper/vectors/{MultiDenseVectorFieldMapperTests.java => RankVectorsFieldMapperTests.java} (87%) rename server/src/test/java/org/elasticsearch/index/mapper/vectors/{MultiDenseVectorFieldTypeTests.java => RankVectorsFieldTypeTests.java} (62%) rename server/src/test/java/org/elasticsearch/index/mapper/vectors/{MultiDenseVectorScriptDocValuesTests.java => RankVectorsScriptDocValuesTests.java} (75%) rename server/src/test/java/org/elasticsearch/script/{MultiVectorScoreScriptUtilsTests.java => RankVectorsScoreScriptUtilsTests.java} (81%) rename server/src/test/java/org/elasticsearch/script/field/vectors/{MultiDenseVectorTests.java => RankVectorsTests.java} (80%) diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt index 875b9a1dac3e8..85dba97a392b4 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt @@ -132,8 +132,8 @@ class org.elasticsearch.script.field.SeqNoDocValuesField @dynamic_type { class org.elasticsearch.script.field.VersionDocValuesField @dynamic_type { } -class org.elasticsearch.script.field.vectors.MultiDenseVector { - MultiDenseVector EMPTY +class org.elasticsearch.script.field.vectors.RankVectors { + RankVectors EMPTY float[] getMagnitudes() Iterator getVectors() @@ -142,9 +142,9 @@ class org.elasticsearch.script.field.vectors.MultiDenseVector { int size() } -class org.elasticsearch.script.field.vectors.MultiDenseVectorDocValuesField { - MultiDenseVector get() - MultiDenseVector get(MultiDenseVector) +class org.elasticsearch.script.field.vectors.RankVectorsDocValuesField { + RankVectors get() + RankVectors get(RankVectors) } class org.elasticsearch.script.field.vectors.DenseVector { diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt index 5a1d8c002aa17..a5118db4876cb 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt @@ -50,7 +50,7 @@ static_import { double cosineSimilarity(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$CosineSimilarity double dotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$DotProduct double hamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$Hamming - double maxSimDotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.MultiVectorScoreScriptUtils$MaxSimDotProduct - double maxSimInvHamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.MultiVectorScoreScriptUtils$MaxSimInvHamming + double maxSimDotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.RankVectorsScoreScriptUtils$MaxSimDotProduct + double maxSimInvHamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.RankVectorsScoreScriptUtils$MaxSimInvHamming } diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt index b2db0d1006d40..4815b9c10e733 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt @@ -123,7 +123,7 @@ class org.elasticsearch.index.mapper.vectors.DenseVectorScriptDocValues { float getMagnitude() } -class org.elasticsearch.index.mapper.vectors.MultiDenseVectorScriptDocValues { +class org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValues { Iterator getVectorValues() float[] getMagnitudes() } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_multi_dense_vector_max_sim.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml similarity index 95% rename from modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_multi_dense_vector_max_sim.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml index 77d4b70cdfcae..7c46fbc9a26a5 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_multi_dense_vector_max_sim.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_rank_vectors_max_sim.yml @@ -3,9 +3,9 @@ setup: capabilities: - method: POST path: /_search - capabilities: [ multi_dense_vector_script_max_sim_with_bugfix ] + capabilities: [ rank_vectors_script_max_sim_with_bugfix ] test_runner_features: capabilities - reason: "Support for multi dense vector max-sim functions capability required" + reason: "Support for rank vectors max-sim functions capability required" - skip: features: headers @@ -18,14 +18,14 @@ setup: mappings: properties: vector: - type: multi_dense_vector + type: rank_vectors dims: 5 byte_vector: - type: multi_dense_vector + type: rank_vectors dims: 5 element_type: byte bit_vector: - type: multi_dense_vector + type: rank_vectors dims: 40 element_type: bit - do: diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_multi_dense_vector_dv_fields_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml similarity index 94% rename from modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_multi_dense_vector_dv_fields_api.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml index 66cb3f3c46fcc..f37e554fca7bf 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_multi_dense_vector_dv_fields_api.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/181_rank_vectors_dv_fields_api.yml @@ -3,9 +3,9 @@ setup: capabilities: - method: POST path: /_search - capabilities: [ multi_dense_vector_script_access ] + capabilities: [ rank_vectors_script_access ] test_runner_features: capabilities - reason: "Support for multi dense vector field script access capability required" + reason: "Support for rank vector field script access capability required" - skip: features: headers @@ -18,14 +18,14 @@ setup: mappings: properties: vector: - type: multi_dense_vector + type: rank_vectors dims: 5 byte_vector: - type: multi_dense_vector + type: rank_vectors dims: 5 element_type: byte bit_vector: - type: multi_dense_vector + type: rank_vectors dims: 40 element_type: bit - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_multi_dense_vector.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml similarity index 88% rename from rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_multi_dense_vector.yml rename to rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml index 80d1d25dfcbd8..ecf34f46c3383 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_multi_dense_vector.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_rank_vectors.yml @@ -3,9 +3,9 @@ setup: capabilities: - method: POST path: /_search - capabilities: [ multi_dense_vector_field_mapper ] + capabilities: [ rank_vectors_field_mapper ] test_runner_features: capabilities - reason: "Support for multi dense vector field mapper capability required" + reason: "Support for rank vectors field mapper capability required" --- "Test create multi-vector field": - do: @@ -15,7 +15,7 @@ setup: mappings: properties: vector1: - type: multi_dense_vector + type: rank_vectors dims: 3 - do: index: @@ -48,7 +48,7 @@ setup: name: type: keyword vector1: - type: multi_dense_vector + type: rank_vectors - do: index: index: test @@ -88,7 +88,7 @@ setup: mappings: properties: vector1: - type: multi_dense_vector + type: rank_vectors - do: catch: bad_request index: @@ -105,7 +105,7 @@ setup: mappings: properties: vector1: - type: multi_dense_vector + type: rank_vectors dims: 3 - do: catch: bad_request @@ -123,7 +123,7 @@ setup: mappings: properties: vector1: - type: multi_dense_vector + type: rank_vectors dims: 3 - do: catch: bad_request diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java similarity index 70% rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorDVLeafFieldData.java rename to server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java index b9716d315f33a..0125d0249ec2b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsDVLeafFieldData.java @@ -15,19 +15,19 @@ import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; -import org.elasticsearch.script.field.vectors.BitMultiDenseVectorDocValuesField; -import org.elasticsearch.script.field.vectors.ByteMultiDenseVectorDocValuesField; -import org.elasticsearch.script.field.vectors.FloatMultiDenseVectorDocValuesField; +import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; import java.io.IOException; -final class MultiVectorDVLeafFieldData implements LeafFieldData { +final class RankVectorsDVLeafFieldData implements LeafFieldData { private final LeafReader reader; private final String field; private final DenseVectorFieldMapper.ElementType elementType; private final int dims; - MultiVectorDVLeafFieldData(LeafReader reader, String field, DenseVectorFieldMapper.ElementType elementType, int dims) { + RankVectorsDVLeafFieldData(LeafReader reader, String field, DenseVectorFieldMapper.ElementType elementType, int dims) { this.reader = reader; this.field = field; this.elementType = elementType; @@ -38,11 +38,11 @@ final class MultiVectorDVLeafFieldData implements LeafFieldData { public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { try { BinaryDocValues values = DocValues.getBinary(reader, field); - BinaryDocValues magnitudeValues = DocValues.getBinary(reader, field + MultiDenseVectorFieldMapper.VECTOR_MAGNITUDES_SUFFIX); + BinaryDocValues magnitudeValues = DocValues.getBinary(reader, field + RankVectorsFieldMapper.VECTOR_MAGNITUDES_SUFFIX); return switch (elementType) { - case BYTE -> new ByteMultiDenseVectorDocValuesField(values, magnitudeValues, name, elementType, dims); - case FLOAT -> new FloatMultiDenseVectorDocValuesField(values, magnitudeValues, name, elementType, dims); - case BIT -> new BitMultiDenseVectorDocValuesField(values, magnitudeValues, name, elementType, dims); + case BYTE -> new ByteRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); + case FLOAT -> new FloatRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); + case BIT -> new BitRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); }; } catch (IOException e) { throw new IllegalStateException("Cannot load doc values for multi-vector field!", e); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java similarity index 90% rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldMapper.java rename to server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java index b23a1f1f66792..d57dbf79b450c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapper.java @@ -51,14 +51,14 @@ import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT_BIT; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.namesToElementType; -public class MultiDenseVectorFieldMapper extends FieldMapper { +public class RankVectorsFieldMapper extends FieldMapper { public static final String VECTOR_MAGNITUDES_SUFFIX = "._magnitude"; - public static final FeatureFlag FEATURE_FLAG = new FeatureFlag("multi_dense_vector"); - public static final String CONTENT_TYPE = "multi_dense_vector"; + public static final FeatureFlag FEATURE_FLAG = new FeatureFlag("rank_vectors"); + public static final String CONTENT_TYPE = "rank_vectors"; - private static MultiDenseVectorFieldMapper toType(FieldMapper in) { - return (MultiDenseVectorFieldMapper) in; + private static RankVectorsFieldMapper toType(FieldMapper in) { + return (RankVectorsFieldMapper) in; } public static class Builder extends FieldMapper.Builder { @@ -122,24 +122,24 @@ protected Parameter[] getParameters() { return new Parameter[] { elementType, dims, meta }; } - public MultiDenseVectorFieldMapper.Builder dimensions(int dimensions) { + public RankVectorsFieldMapper.Builder dimensions(int dimensions) { this.dims.setValue(dimensions); return this; } - public MultiDenseVectorFieldMapper.Builder elementType(DenseVectorFieldMapper.ElementType elementType) { + public RankVectorsFieldMapper.Builder elementType(DenseVectorFieldMapper.ElementType elementType) { this.elementType.setValue(elementType); return this; } @Override - public MultiDenseVectorFieldMapper build(MapperBuilderContext context) { + public RankVectorsFieldMapper build(MapperBuilderContext context) { // Validate again here because the dimensions or element type could have been set programmatically, // which affects index option validity validate(); - return new MultiDenseVectorFieldMapper( + return new RankVectorsFieldMapper( leafName(), - new MultiDenseVectorFieldType( + new RankVectorsFieldType( context.buildFullName(leafName()), elementType.getValue(), dims.getValue(), @@ -153,16 +153,16 @@ public MultiDenseVectorFieldMapper build(MapperBuilderContext context) { } public static final TypeParser PARSER = new TypeParser( - (n, c) -> new MultiDenseVectorFieldMapper.Builder(n, c.indexVersionCreated()), + (n, c) -> new RankVectorsFieldMapper.Builder(n, c.indexVersionCreated()), notInMultiFields(CONTENT_TYPE) ); - public static final class MultiDenseVectorFieldType extends SimpleMappedFieldType { + public static final class RankVectorsFieldType extends SimpleMappedFieldType { private final DenseVectorFieldMapper.ElementType elementType; private final Integer dims; private final IndexVersion indexCreatedVersion; - public MultiDenseVectorFieldType( + public RankVectorsFieldType( String name, DenseVectorFieldMapper.ElementType elementType, Integer dims, @@ -207,7 +207,7 @@ public boolean isAggregatable() { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - return new MultiVectorIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, indexCreatedVersion, dims, elementType); + return new RankVectorsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, indexCreatedVersion, dims, elementType); } @Override @@ -231,19 +231,14 @@ DenseVectorFieldMapper.ElementType getElementType() { private final IndexVersion indexCreatedVersion; - private MultiDenseVectorFieldMapper( - String simpleName, - MappedFieldType fieldType, - BuilderParams params, - IndexVersion indexCreatedVersion - ) { + private RankVectorsFieldMapper(String simpleName, MappedFieldType fieldType, BuilderParams params, IndexVersion indexCreatedVersion) { super(simpleName, fieldType, params); this.indexCreatedVersion = indexCreatedVersion; } @Override - public MultiDenseVectorFieldType fieldType() { - return (MultiDenseVectorFieldType) super.fieldType(); + public RankVectorsFieldType fieldType() { + return (RankVectorsFieldType) super.fieldType(); } @Override @@ -282,14 +277,14 @@ public void parse(DocumentParserContext context) throws IOException { ); } } - MultiDenseVectorFieldType updatedFieldType = new MultiDenseVectorFieldType( + RankVectorsFieldType updatedFieldType = new RankVectorsFieldType( fieldType().name(), fieldType().elementType, currentDims, indexCreatedVersion, fieldType().meta() ); - Mapper update = new MultiDenseVectorFieldMapper(leafName(), updatedFieldType, builderParams, indexCreatedVersion); + Mapper update = new RankVectorsFieldMapper(leafName(), updatedFieldType, builderParams, indexCreatedVersion); context.addDynamicMapper(update); return; } @@ -371,12 +366,12 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new MultiDenseVectorFieldMapper.Builder(leafName(), indexCreatedVersion).init(this); + return new RankVectorsFieldMapper.Builder(leafName(), indexCreatedVersion).init(this); } @Override protected SyntheticSourceSupport syntheticSourceSupport() { - return new SyntheticSourceSupport.Native(new MultiDenseVectorFieldMapper.DocValuesSyntheticFieldLoader()); + return new SyntheticSourceSupport.Native(new RankVectorsFieldMapper.DocValuesSyntheticFieldLoader()); } private class DocValuesSyntheticFieldLoader extends SourceLoader.DocValuesBasedSyntheticFieldLoader { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java similarity index 87% rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorIndexFieldData.java rename to server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java index 44a666e25a611..7f54d2b9a8ad8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsIndexFieldData.java @@ -22,14 +22,14 @@ import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; -public class MultiVectorIndexFieldData implements IndexFieldData { +public class RankVectorsIndexFieldData implements IndexFieldData { protected final String fieldName; protected final ValuesSourceType valuesSourceType; private final int dims; private final IndexVersion indexVersion; private final DenseVectorFieldMapper.ElementType elementType; - public MultiVectorIndexFieldData( + public RankVectorsIndexFieldData( String fieldName, int dims, ValuesSourceType valuesSourceType, @@ -54,19 +54,19 @@ public ValuesSourceType getValuesSourceType() { } @Override - public MultiVectorDVLeafFieldData load(LeafReaderContext context) { - return new MultiVectorDVLeafFieldData(context.reader(), fieldName, elementType, dims); + public RankVectorsDVLeafFieldData load(LeafReaderContext context) { + return new RankVectorsDVLeafFieldData(context.reader(), fieldName, elementType, dims); } @Override - public MultiVectorDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public RankVectorsDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception { return load(context); } @Override public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { throw new IllegalArgumentException( - "Field [" + fieldName + "] of type [" + MultiDenseVectorFieldMapper.CONTENT_TYPE + "] doesn't support sort" + "Field [" + fieldName + "] of type [" + RankVectorsFieldMapper.CONTENT_TYPE + "] doesn't support sort" ); } @@ -108,7 +108,7 @@ public Builder( @Override public IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { - return new MultiVectorIndexFieldData(name, dims, valuesSourceType, indexVersion, elementType); + return new RankVectorsIndexFieldData(name, dims, valuesSourceType, indexVersion, elementType); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValues.java similarity index 70% rename from server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValues.java rename to server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValues.java index a91960832239f..e663df86c67ca 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValues.java @@ -11,18 +11,18 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.script.field.vectors.MultiDenseVector; +import org.elasticsearch.script.field.vectors.RankVectors; import java.util.Iterator; -public class MultiDenseVectorScriptDocValues extends ScriptDocValues { +public class RankVectorsScriptDocValues extends ScriptDocValues { - public static final String MISSING_VECTOR_FIELD_MESSAGE = "A document doesn't have a value for a multi-vector field!"; + public static final String MISSING_VECTOR_FIELD_MESSAGE = "A document doesn't have a value for a rank-vectors field!"; private final int dims; - protected final MultiDenseVectorSupplier dvSupplier; + protected final RankVectorsSupplier dvSupplier; - public MultiDenseVectorScriptDocValues(MultiDenseVectorSupplier supplier, int dims) { + public RankVectorsScriptDocValues(RankVectorsSupplier supplier, int dims) { super(supplier); this.dvSupplier = supplier; this.dims = dims; @@ -32,8 +32,8 @@ public int dims() { return dims; } - private MultiDenseVector getCheckedVector() { - MultiDenseVector vector = dvSupplier.getInternal(); + private RankVectors getCheckedVector() { + RankVectors vector = dvSupplier.getInternal(); if (vector == null) { throw new IllegalArgumentException(MISSING_VECTOR_FIELD_MESSAGE); } @@ -41,7 +41,7 @@ private MultiDenseVector getCheckedVector() { } /** - * Get multi-dense vector's value as an array of floats + * Get rank-vectors's value as an array of floats */ public Iterator getVectorValues() { return getCheckedVector().getVectors(); @@ -57,25 +57,25 @@ public float[] getMagnitudes() { @Override public BytesRef get(int index) { throw new UnsupportedOperationException( - "accessing a multi-vector field's value through 'get' or 'value' is not supported, use 'vectorValues' or 'magnitudes' instead." + "accessing a rank-vectors field's value through 'get' or 'value' is not supported, use 'vectorValues' or 'magnitudes' instead." ); } @Override public int size() { - MultiDenseVector mdv = dvSupplier.getInternal(); + RankVectors mdv = dvSupplier.getInternal(); if (mdv != null) { return mdv.size(); } return 0; } - public interface MultiDenseVectorSupplier extends Supplier { + public interface RankVectorsSupplier extends Supplier { @Override default BytesRef getInternal(int index) { throw new UnsupportedOperationException(); } - MultiDenseVector getInternal(); + RankVectors getInternal(); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorEncoderDecoder.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorEncoderDecoder.java index 3db2d164846bd..54b369ab1f377 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorEncoderDecoder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorEncoderDecoder.java @@ -94,14 +94,4 @@ public static float[] getMultiMagnitudes(BytesRef magnitudes) { return multiMagnitudes; } - public static void decodeMultiDenseVector(BytesRef vectorBR, int numVectors, float[][] multiVectorValue) { - if (vectorBR == null) { - throw new IllegalArgumentException(MultiDenseVectorScriptDocValues.MISSING_VECTOR_FIELD_MESSAGE); - } - FloatBuffer fb = ByteBuffer.wrap(vectorBR.bytes, vectorBR.offset, vectorBR.length).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer(); - for (int i = 0; i < numVectors; i++) { - fb.get(multiVectorValue[i]); - } - } - } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 340bff4e1c852..3dc25b058b1d6 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -67,7 +67,7 @@ import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; -import org.elasticsearch.index.mapper.vectors.MultiDenseVectorFieldMapper; +import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseSyncAction; @@ -211,8 +211,8 @@ public static Map getMappers(List mappe mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, DenseVectorFieldMapper.PARSER); mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, SparseVectorFieldMapper.PARSER); - if (MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()) { - mappers.put(MultiDenseVectorFieldMapper.CONTENT_TYPE, MultiDenseVectorFieldMapper.PARSER); + if (RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()) { + mappers.put(RankVectorsFieldMapper.CONTENT_TYPE, RankVectorsFieldMapper.PARSER); } for (MapperPlugin mapperPlugin : mapperPlugins) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index 57980321bdc3d..c9d9569abe93f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -10,7 +10,7 @@ package org.elasticsearch.rest.action.search; import org.elasticsearch.Build; -import org.elasticsearch.index.mapper.vectors.MultiDenseVectorFieldMapper; +import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import java.util.HashSet; import java.util.Set; @@ -34,14 +34,14 @@ private SearchCapabilities() {} private static final String TRANSFORM_RANK_RRF_TO_RETRIEVER = "transform_rank_rrf_to_retriever"; /** Support kql query. */ private static final String KQL_QUERY_SUPPORTED = "kql_query"; - /** Support multi-dense-vector field mapper. */ - private static final String MULTI_DENSE_VECTOR_FIELD_MAPPER = "multi_dense_vector_field_mapper"; + /** Support rank-vectors field mapper. */ + private static final String RANK_VECTORS_FIELD_MAPPER = "rank_vectors_field_mapper"; /** Support propagating nested retrievers' inner_hits to top-level compound retrievers . */ private static final String NESTED_RETRIEVER_INNER_HITS_SUPPORT = "nested_retriever_inner_hits_support"; - /** Support multi-dense-vector script field access. */ - private static final String MULTI_DENSE_VECTOR_SCRIPT_ACCESS = "multi_dense_vector_script_access"; - /** Initial support for multi-dense-vector maxSim functions access. */ - private static final String MULTI_DENSE_VECTOR_SCRIPT_MAX_SIM = "multi_dense_vector_script_max_sim_with_bugfix"; + /** Support rank-vectors script field access. */ + private static final String RANK_VECTORS_SCRIPT_ACCESS = "rank_vectors_script_access"; + /** Initial support for rank-vectors maxSim functions access. */ + private static final String RANK_VECTORS_SCRIPT_MAX_SIM = "rank_vectors_script_max_sim_with_bugfix"; private static final String RANDOM_SAMPLER_WITH_SCORED_SUBAGGS = "random_sampler_with_scored_subaggs"; private static final String OPTIMIZED_SCALAR_QUANTIZATION_BBQ = "optimized_scalar_quantization_bbq"; @@ -57,10 +57,10 @@ private SearchCapabilities() {} capabilities.add(NESTED_RETRIEVER_INNER_HITS_SUPPORT); capabilities.add(RANDOM_SAMPLER_WITH_SCORED_SUBAGGS); capabilities.add(OPTIMIZED_SCALAR_QUANTIZATION_BBQ); - if (MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()) { - capabilities.add(MULTI_DENSE_VECTOR_FIELD_MAPPER); - capabilities.add(MULTI_DENSE_VECTOR_SCRIPT_ACCESS); - capabilities.add(MULTI_DENSE_VECTOR_SCRIPT_MAX_SIM); + if (RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()) { + capabilities.add(RANK_VECTORS_FIELD_MAPPER); + capabilities.add(RANK_VECTORS_SCRIPT_ACCESS); + capabilities.add(RANK_VECTORS_SCRIPT_MAX_SIM); } if (Build.current().isSnapshot()) { capabilities.add(KQL_QUERY_SUPPORTED); diff --git a/server/src/main/java/org/elasticsearch/script/MultiVectorScoreScriptUtils.java b/server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java similarity index 85% rename from server/src/main/java/org/elasticsearch/script/MultiVectorScoreScriptUtils.java rename to server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java index 136c5e7b57d4b..2d11641cb5aa7 100644 --- a/server/src/main/java/org/elasticsearch/script/MultiVectorScoreScriptUtils.java +++ b/server/src/main/java/org/elasticsearch/script/RankVectorsScoreScriptUtils.java @@ -12,19 +12,19 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.script.field.vectors.DenseVector; -import org.elasticsearch.script.field.vectors.MultiDenseVectorDocValuesField; +import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; import java.io.IOException; import java.util.HexFormat; import java.util.List; -public class MultiVectorScoreScriptUtils { +public class RankVectorsScoreScriptUtils { - public static class MultiDenseVectorFunction { + public static class RankVectorsFunction { protected final ScoreScript scoreScript; - protected final MultiDenseVectorDocValuesField field; + protected final RankVectorsDocValuesField field; - public MultiDenseVectorFunction(ScoreScript scoreScript, MultiDenseVectorDocValuesField field) { + public RankVectorsFunction(ScoreScript scoreScript, RankVectorsDocValuesField field) { this.scoreScript = scoreScript; this.field = field; } @@ -41,7 +41,7 @@ void setNextVector() { } } - public static class ByteMultiDenseVectorFunction extends MultiDenseVectorFunction { + public static class ByteRankVectorsFunction extends RankVectorsFunction { protected final byte[][] queryVector; /** @@ -51,7 +51,7 @@ public static class ByteMultiDenseVectorFunction extends MultiDenseVectorFunctio * @param field The vector field. * @param queryVector The query vector. */ - public ByteMultiDenseVectorFunction(ScoreScript scoreScript, MultiDenseVectorDocValuesField field, List> queryVector) { + public ByteRankVectorsFunction(ScoreScript scoreScript, RankVectorsDocValuesField field, List> queryVector) { super(scoreScript, field); if (queryVector.isEmpty()) { throw new IllegalArgumentException("The query vector is empty."); @@ -84,13 +84,13 @@ public ByteMultiDenseVectorFunction(ScoreScript scoreScript, MultiDenseVectorDoc * @param field The vector field. * @param queryVector The query vector. */ - public ByteMultiDenseVectorFunction(ScoreScript scoreScript, MultiDenseVectorDocValuesField field, byte[][] queryVector) { + public ByteRankVectorsFunction(ScoreScript scoreScript, RankVectorsDocValuesField field, byte[][] queryVector) { super(scoreScript, field); this.queryVector = queryVector; } } - public static class FloatMultiDenseVectorFunction extends MultiDenseVectorFunction { + public static class FloatRankVectorsFunction extends RankVectorsFunction { protected final float[][] queryVector; /** @@ -100,11 +100,7 @@ public static class FloatMultiDenseVectorFunction extends MultiDenseVectorFuncti * @param field The vector field. * @param queryVector The query vector. */ - public FloatMultiDenseVectorFunction( - ScoreScript scoreScript, - MultiDenseVectorDocValuesField field, - List> queryVector - ) { + public FloatRankVectorsFunction(ScoreScript scoreScript, RankVectorsDocValuesField field, List> queryVector) { super(scoreScript, field); if (queryVector.isEmpty()) { throw new IllegalArgumentException("The query vector is empty."); @@ -133,13 +129,13 @@ public interface MaxSimInvHammingDistanceInterface { float maxSimInvHamming(); } - public static class ByteMaxSimInvHammingDistance extends ByteMultiDenseVectorFunction implements MaxSimInvHammingDistanceInterface { + public static class ByteMaxSimInvHammingDistance extends ByteRankVectorsFunction implements MaxSimInvHammingDistanceInterface { - public ByteMaxSimInvHammingDistance(ScoreScript scoreScript, MultiDenseVectorDocValuesField field, List> queryVector) { + public ByteMaxSimInvHammingDistance(ScoreScript scoreScript, RankVectorsDocValuesField field, List> queryVector) { super(scoreScript, field, queryVector); } - public ByteMaxSimInvHammingDistance(ScoreScript scoreScript, MultiDenseVectorDocValuesField field, byte[][] queryVector) { + public ByteMaxSimInvHammingDistance(ScoreScript scoreScript, RankVectorsDocValuesField field, byte[][] queryVector) { super(scoreScript, field, queryVector); } @@ -183,7 +179,7 @@ public static final class MaxSimInvHamming { private final MaxSimInvHammingDistanceInterface function; public MaxSimInvHamming(ScoreScript scoreScript, Object queryVector, String fieldName) { - MultiDenseVectorDocValuesField field = (MultiDenseVectorDocValuesField) scoreScript.field(fieldName); + RankVectorsDocValuesField field = (RankVectorsDocValuesField) scoreScript.field(fieldName); if (field.getElementType() == DenseVectorFieldMapper.ElementType.FLOAT) { throw new IllegalArgumentException("hamming distance is only supported for byte or bit vectors"); } @@ -205,11 +201,11 @@ public interface MaxSimDotProductInterface { double maxSimDotProduct(); } - public static class MaxSimBitDotProduct extends MultiDenseVectorFunction implements MaxSimDotProductInterface { + public static class MaxSimBitDotProduct extends RankVectorsFunction implements MaxSimDotProductInterface { private final byte[][] byteQueryVector; private final float[][] floatQueryVector; - public MaxSimBitDotProduct(ScoreScript scoreScript, MultiDenseVectorDocValuesField field, byte[][] queryVector) { + public MaxSimBitDotProduct(ScoreScript scoreScript, RankVectorsDocValuesField field, byte[][] queryVector) { super(scoreScript, field); if (field.getElementType() != DenseVectorFieldMapper.ElementType.BIT) { throw new IllegalArgumentException("Cannot calculate bit dot product for non-bit vectors"); @@ -230,7 +226,7 @@ public MaxSimBitDotProduct(ScoreScript scoreScript, MultiDenseVectorDocValuesFie this.floatQueryVector = null; } - public MaxSimBitDotProduct(ScoreScript scoreScript, MultiDenseVectorDocValuesField field, List> queryVector) { + public MaxSimBitDotProduct(ScoreScript scoreScript, RankVectorsDocValuesField field, List> queryVector) { super(scoreScript, field); if (queryVector.isEmpty()) { throw new IllegalArgumentException("The query vector is empty."); @@ -304,13 +300,13 @@ public double maxSimDotProduct() { } } - public static class MaxSimByteDotProduct extends ByteMultiDenseVectorFunction implements MaxSimDotProductInterface { + public static class MaxSimByteDotProduct extends ByteRankVectorsFunction implements MaxSimDotProductInterface { - public MaxSimByteDotProduct(ScoreScript scoreScript, MultiDenseVectorDocValuesField field, List> queryVector) { + public MaxSimByteDotProduct(ScoreScript scoreScript, RankVectorsDocValuesField field, List> queryVector) { super(scoreScript, field, queryVector); } - public MaxSimByteDotProduct(ScoreScript scoreScript, MultiDenseVectorDocValuesField field, byte[][] queryVector) { + public MaxSimByteDotProduct(ScoreScript scoreScript, RankVectorsDocValuesField field, byte[][] queryVector) { super(scoreScript, field, queryVector); } @@ -320,9 +316,9 @@ public double maxSimDotProduct() { } } - public static class MaxSimFloatDotProduct extends FloatMultiDenseVectorFunction implements MaxSimDotProductInterface { + public static class MaxSimFloatDotProduct extends FloatRankVectorsFunction implements MaxSimDotProductInterface { - public MaxSimFloatDotProduct(ScoreScript scoreScript, MultiDenseVectorDocValuesField field, List> queryVector) { + public MaxSimFloatDotProduct(ScoreScript scoreScript, RankVectorsDocValuesField field, List> queryVector) { super(scoreScript, field, queryVector); } @@ -338,7 +334,7 @@ public static final class MaxSimDotProduct { @SuppressWarnings("unchecked") public MaxSimDotProduct(ScoreScript scoreScript, Object queryVector, String fieldName) { - MultiDenseVectorDocValuesField field = (MultiDenseVectorDocValuesField) scoreScript.field(fieldName); + RankVectorsDocValuesField field = (RankVectorsDocValuesField) scoreScript.field(fieldName); function = switch (field.getElementType()) { case BIT -> { BytesOrList bytesOrList = parseBytes(queryVector); diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/BitMultiDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/BitRankVectors.java similarity index 94% rename from server/src/main/java/org/elasticsearch/script/field/vectors/BitMultiDenseVector.java rename to server/src/main/java/org/elasticsearch/script/field/vectors/BitRankVectors.java index 7805816090d51..0e2984c2a7dff 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/BitMultiDenseVector.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/BitRankVectors.java @@ -15,8 +15,8 @@ import java.util.Arrays; -public class BitMultiDenseVector extends ByteMultiDenseVector { - public BitMultiDenseVector(VectorIterator vectorValues, BytesRef magnitudesBytes, int numVecs, int dims) { +public class BitRankVectors extends ByteRankVectors { + public BitRankVectors(VectorIterator vectorValues, BytesRef magnitudesBytes, int numVecs, int dims) { super(vectorValues, magnitudesBytes, numVecs, dims); } diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/BitMultiDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/BitRankVectorsDocValuesField.java similarity index 64% rename from server/src/main/java/org/elasticsearch/script/field/vectors/BitMultiDenseVectorDocValuesField.java rename to server/src/main/java/org/elasticsearch/script/field/vectors/BitRankVectorsDocValuesField.java index 35a43eabb8f0c..6d38621440fbf 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/BitMultiDenseVectorDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/BitRankVectorsDocValuesField.java @@ -12,20 +12,14 @@ import org.apache.lucene.index.BinaryDocValues; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; -public class BitMultiDenseVectorDocValuesField extends ByteMultiDenseVectorDocValuesField { +public class BitRankVectorsDocValuesField extends ByteRankVectorsDocValuesField { - public BitMultiDenseVectorDocValuesField( - BinaryDocValues input, - BinaryDocValues magnitudes, - String name, - ElementType elementType, - int dims - ) { + public BitRankVectorsDocValuesField(BinaryDocValues input, BinaryDocValues magnitudes, String name, ElementType elementType, int dims) { super(input, magnitudes, name, elementType, dims / 8); } @Override - protected MultiDenseVector getVector() { - return new BitMultiDenseVector(vectorValue, magnitudesValue, numVecs, dims); + protected RankVectors getVector() { + return new BitRankVectors(vectorValue, magnitudesValue, numVecs, dims); } } diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteMultiDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectors.java similarity index 95% rename from server/src/main/java/org/elasticsearch/script/field/vectors/ByteMultiDenseVector.java rename to server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectors.java index 5e9d3e05746c8..f8e82046037c4 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteMultiDenseVector.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectors.java @@ -16,7 +16,7 @@ import java.util.Arrays; import java.util.Iterator; -public class ByteMultiDenseVector implements MultiDenseVector { +public class ByteRankVectors implements RankVectors { protected final VectorIterator vectorValues; protected final int numVecs; @@ -25,7 +25,7 @@ public class ByteMultiDenseVector implements MultiDenseVector { private float[] magnitudes; private final BytesRef magnitudesBytes; - public ByteMultiDenseVector(VectorIterator vectorValues, BytesRef magnitudesBytes, int numVecs, int dims) { + public ByteRankVectors(VectorIterator vectorValues, BytesRef magnitudesBytes, int numVecs, int dims) { assert magnitudesBytes.length == numVecs * Float.BYTES; this.vectorValues = vectorValues; this.numVecs = numVecs; diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteMultiDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java similarity index 85% rename from server/src/main/java/org/elasticsearch/script/field/vectors/ByteMultiDenseVectorDocValuesField.java rename to server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java index d45c5b85137f5..db81bb6ebe1cb 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteMultiDenseVectorDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteRankVectorsDocValuesField.java @@ -12,12 +12,12 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; -import org.elasticsearch.index.mapper.vectors.MultiDenseVectorScriptDocValues; +import org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValues; import java.io.IOException; import java.util.Iterator; -public class ByteMultiDenseVectorDocValuesField extends MultiDenseVectorDocValuesField { +public class ByteRankVectorsDocValuesField extends RankVectorsDocValuesField { protected final BinaryDocValues input; private final BinaryDocValues magnitudes; @@ -29,7 +29,7 @@ public class ByteMultiDenseVectorDocValuesField extends MultiDenseVectorDocValue protected BytesRef magnitudesValue; private byte[] buffer; - public ByteMultiDenseVectorDocValuesField( + public ByteRankVectorsDocValuesField( BinaryDocValues input, BinaryDocValues magnitudes, String name, @@ -63,25 +63,25 @@ public void setNextDocId(int docId) throws IOException { } @Override - public MultiDenseVectorScriptDocValues toScriptDocValues() { - return new MultiDenseVectorScriptDocValues(this, dims); + public RankVectorsScriptDocValues toScriptDocValues() { + return new RankVectorsScriptDocValues(this, dims); } - protected MultiDenseVector getVector() { - return new ByteMultiDenseVector(vectorValue, magnitudesValue, numVecs, dims); + protected RankVectors getVector() { + return new ByteRankVectors(vectorValue, magnitudesValue, numVecs, dims); } @Override - public MultiDenseVector get() { + public RankVectors get() { if (isEmpty()) { - return MultiDenseVector.EMPTY; + return RankVectors.EMPTY; } decodeVectorIfNecessary(); return getVector(); } @Override - public MultiDenseVector get(MultiDenseVector defaultValue) { + public RankVectors get(RankVectors defaultValue) { if (isEmpty()) { return defaultValue; } @@ -90,7 +90,7 @@ public MultiDenseVector get(MultiDenseVector defaultValue) { } @Override - public MultiDenseVector getInternal() { + public RankVectors getInternal() { return get(null); } diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatMultiDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectors.java similarity index 93% rename from server/src/main/java/org/elasticsearch/script/field/vectors/FloatMultiDenseVector.java rename to server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectors.java index 9c2f7eb6a86d4..3ad5e53c047ae 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatMultiDenseVector.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectors.java @@ -17,7 +17,7 @@ import static org.elasticsearch.index.mapper.vectors.VectorEncoderDecoder.getMultiMagnitudes; -public class FloatMultiDenseVector implements MultiDenseVector { +public class FloatRankVectors implements RankVectors { private final BytesRef magnitudes; private float[] magnitudesArray = null; @@ -25,7 +25,7 @@ public class FloatMultiDenseVector implements MultiDenseVector { private final int numVectors; private final VectorIterator vectorValues; - public FloatMultiDenseVector(VectorIterator decodedDocVector, BytesRef magnitudes, int numVectors, int dims) { + public FloatRankVectors(VectorIterator decodedDocVector, BytesRef magnitudes, int numVectors, int dims) { assert magnitudes.length == numVectors * Float.BYTES; this.vectorValues = decodedDocVector; this.magnitudes = magnitudes; diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatMultiDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java similarity index 85% rename from server/src/main/java/org/elasticsearch/script/field/vectors/FloatMultiDenseVectorDocValuesField.java rename to server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java index c7ac7842afd96..39bc1e621113b 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/FloatMultiDenseVectorDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/FloatRankVectorsDocValuesField.java @@ -12,7 +12,7 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; -import org.elasticsearch.index.mapper.vectors.MultiDenseVectorScriptDocValues; +import org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValues; import java.io.IOException; import java.nio.ByteBuffer; @@ -20,7 +20,7 @@ import java.nio.FloatBuffer; import java.util.Iterator; -public class FloatMultiDenseVectorDocValuesField extends MultiDenseVectorDocValuesField { +public class FloatRankVectorsDocValuesField extends RankVectorsDocValuesField { private final BinaryDocValues input; private final BinaryDocValues magnitudes; @@ -32,7 +32,7 @@ public class FloatMultiDenseVectorDocValuesField extends MultiDenseVectorDocValu private int numVectors; private float[] buffer; - public FloatMultiDenseVectorDocValuesField( + public FloatRankVectorsDocValuesField( BinaryDocValues input, BinaryDocValues magnitudes, String name, @@ -66,8 +66,8 @@ public void setNextDocId(int docId) throws IOException { } @Override - public MultiDenseVectorScriptDocValues toScriptDocValues() { - return new MultiDenseVectorScriptDocValues(this, dims); + public RankVectorsScriptDocValues toScriptDocValues() { + return new RankVectorsScriptDocValues(this, dims); } @Override @@ -76,25 +76,25 @@ public boolean isEmpty() { } @Override - public MultiDenseVector get() { + public RankVectors get() { if (isEmpty()) { - return MultiDenseVector.EMPTY; + return RankVectors.EMPTY; } decodeVectorIfNecessary(); - return new FloatMultiDenseVector(vectorValues, magnitudesValue, numVectors, dims); + return new FloatRankVectors(vectorValues, magnitudesValue, numVectors, dims); } @Override - public MultiDenseVector get(MultiDenseVector defaultValue) { + public RankVectors get(RankVectors defaultValue) { if (isEmpty()) { return defaultValue; } decodeVectorIfNecessary(); - return new FloatMultiDenseVector(vectorValues, magnitudesValue, numVectors, dims); + return new FloatRankVectors(vectorValues, magnitudesValue, numVectors, dims); } @Override - public MultiDenseVector getInternal() { + public RankVectors getInternal() { return get(null); } diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/MultiDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/RankVectors.java similarity index 92% rename from server/src/main/java/org/elasticsearch/script/field/vectors/MultiDenseVector.java rename to server/src/main/java/org/elasticsearch/script/field/vectors/RankVectors.java index 7d948cf5a74fa..ec0157c2708c8 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/MultiDenseVector.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/RankVectors.java @@ -11,7 +11,7 @@ import java.util.Iterator; -public interface MultiDenseVector { +public interface RankVectors { default void checkDimensions(int qvDims) { checkDimensions(getDims(), qvDims); @@ -45,9 +45,9 @@ private static String badQueryVectorType(Object queryVector) { return "Cannot use vector [" + queryVector + "] with class [" + queryVector.getClass().getName() + "] as query vector"; } - MultiDenseVector EMPTY = new MultiDenseVector() { - public static final String MISSING_VECTOR_FIELD_MESSAGE = "Multi Dense vector value missing for a field," - + " use isEmpty() to check for a missing vector value"; + RankVectors EMPTY = new RankVectors() { + public static final String MISSING_VECTOR_FIELD_MESSAGE = "rank-vectors value missing for a field," + + " use isEmpty() to check for a missing value"; @Override public Iterator getVectors() { diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/MultiDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/RankVectorsDocValuesField.java similarity index 68% rename from server/src/main/java/org/elasticsearch/script/field/vectors/MultiDenseVectorDocValuesField.java rename to server/src/main/java/org/elasticsearch/script/field/vectors/RankVectorsDocValuesField.java index 61ae4304683c8..2362561ea88c5 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/MultiDenseVectorDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/RankVectorsDocValuesField.java @@ -9,7 +9,7 @@ package org.elasticsearch.script.field.vectors; -import org.elasticsearch.index.mapper.vectors.MultiDenseVectorScriptDocValues; +import org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValues; import org.elasticsearch.script.field.AbstractScriptFieldFactory; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; import org.elasticsearch.script.field.Field; @@ -18,15 +18,15 @@ import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; -public abstract class MultiDenseVectorDocValuesField extends AbstractScriptFieldFactory +public abstract class RankVectorsDocValuesField extends AbstractScriptFieldFactory implements - Field, + Field, DocValuesScriptFieldFactory, - MultiDenseVectorScriptDocValues.MultiDenseVectorSupplier { + RankVectorsScriptDocValues.RankVectorsSupplier { protected final String name; protected final ElementType elementType; - public MultiDenseVectorDocValuesField(String name, ElementType elementType) { + public RankVectorsDocValuesField(String name, ElementType elementType) { this.name = name; this.elementType = elementType; } @@ -43,15 +43,15 @@ public ElementType getElementType() { /** * Get the DenseVector for a document if one exists, DenseVector.EMPTY otherwise */ - public abstract MultiDenseVector get(); + public abstract RankVectors get(); - public abstract MultiDenseVector get(MultiDenseVector defaultValue); + public abstract RankVectors get(RankVectors defaultValue); - public abstract MultiDenseVectorScriptDocValues toScriptDocValues(); + public abstract RankVectorsScriptDocValues toScriptDocValues(); // DenseVector fields are single valued, so Iterable does not make sense. @Override - public Iterator iterator() { - throw new UnsupportedOperationException("Cannot iterate over single valued multi_dense_vector field, use get() instead"); + public Iterator iterator() { + throw new UnsupportedOperationException("Cannot iterate over single valued rank_vectors field, use get() instead"); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java similarity index 87% rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldMapperTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java index 6a890328732ca..e81c28cbcc444 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldMapperTests.java @@ -51,17 +51,17 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class MultiDenseVectorFieldMapperTests extends MapperTestCase { +public class RankVectorsFieldMapperTests extends MapperTestCase { @BeforeClass public static void setup() { - assumeTrue("Requires multi-dense vector support", MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()); + assumeTrue("Requires rank vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); } private final ElementType elementType; private final int dims; - public MultiDenseVectorFieldMapperTests() { + public RankVectorsFieldMapperTests() { this.elementType = randomFrom(ElementType.BYTE, ElementType.FLOAT, ElementType.BIT); this.dims = ElementType.BIT == elementType ? 4 * Byte.SIZE : 4; } @@ -77,7 +77,7 @@ protected void minimalMapping(XContentBuilder b, IndexVersion indexVersion) thro } private void indexMapping(XContentBuilder b, IndexVersion indexVersion) throws IOException { - b.field("type", "multi_dense_vector").field("dims", dims); + b.field("type", "rank_vectors").field("dims", dims); if (elementType != ElementType.FLOAT) { b.field("element_type", elementType.toString()); } @@ -95,23 +95,23 @@ protected Object getSampleValueForDocument() { protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck( "dims", - fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims)), - fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims + 8)) + fieldMapping(b -> b.field("type", "rank_vectors").field("dims", dims)), + fieldMapping(b -> b.field("type", "rank_vectors").field("dims", dims + 8)) ); checker.registerConflictCheck( "element_type", - fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims).field("element_type", "byte")), - fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims).field("element_type", "float")) + fieldMapping(b -> b.field("type", "rank_vectors").field("dims", dims).field("element_type", "byte")), + fieldMapping(b -> b.field("type", "rank_vectors").field("dims", dims).field("element_type", "float")) ); checker.registerConflictCheck( "element_type", - fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims).field("element_type", "float")), - fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims * 8).field("element_type", "bit")) + fieldMapping(b -> b.field("type", "rank_vectors").field("dims", dims).field("element_type", "float")), + fieldMapping(b -> b.field("type", "rank_vectors").field("dims", dims * 8).field("element_type", "bit")) ); checker.registerConflictCheck( "element_type", - fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims).field("element_type", "byte")), - fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims * 8).field("element_type", "bit")) + fieldMapping(b -> b.field("type", "rank_vectors").field("dims", dims).field("element_type", "byte")), + fieldMapping(b -> b.field("type", "rank_vectors").field("dims", dims * 8).field("element_type", "bit")) ); } @@ -127,7 +127,7 @@ protected boolean supportsIgnoreMalformed() { @Override protected void assertSearchable(MappedFieldType fieldType) { - assertThat(fieldType, instanceOf(MultiDenseVectorFieldMapper.MultiDenseVectorFieldType.class)); + assertThat(fieldType, instanceOf(RankVectorsFieldMapper.RankVectorsFieldType.class)); assertFalse(fieldType.isIndexed()); assertFalse(fieldType.isSearchable()); } @@ -147,7 +147,7 @@ public void testAggregatableConsistency() {} public void testDims() { { Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> { - b.field("type", "multi_dense_vector"); + b.field("type", "rank_vectors"); b.field("dims", 0); }))); assertThat( @@ -158,7 +158,7 @@ public void testDims() { // test max limit for non-indexed vectors { Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> { - b.field("type", "multi_dense_vector"); + b.field("type", "rank_vectors"); b.field("dims", 5000); }))); assertThat( @@ -171,14 +171,14 @@ public void testDims() { public void testMergeDims() throws IOException { XContentBuilder mapping = mapping(b -> { b.startObject("field"); - b.field("type", "multi_dense_vector"); + b.field("type", "rank_vectors"); b.endObject(); }); MapperService mapperService = createMapperService(mapping); mapping = mapping(b -> { b.startObject("field"); - b.field("type", "multi_dense_vector").field("dims", dims); + b.field("type", "rank_vectors").field("dims", dims); b.endObject(); }); merge(mapperService, mapping); @@ -190,14 +190,14 @@ public void testMergeDims() throws IOException { public void testLargeDimsBit() throws IOException { createMapperService(fieldMapping(b -> { - b.field("type", "multi_dense_vector"); + b.field("type", "rank_vectors"); b.field("dims", 1024 * Byte.SIZE); b.field("element_type", ElementType.BIT.toString()); })); } public void testNonIndexedVector() throws Exception { - DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", 3))); + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "rank_vectors").field("dims", 3))); float[][] validVectors = { { -12.1f, 100.7f, -4 }, { 42f, .05f, -1f } }; double[] dotProduct = new double[2]; @@ -234,7 +234,7 @@ public void testNonIndexedVector() throws Exception { .asFloatBuffer(); fb.get(decodedValues[i]); } - List magFields = doc1.rootDoc().getFields("field" + MultiDenseVectorFieldMapper.VECTOR_MAGNITUDES_SUFFIX); + List magFields = doc1.rootDoc().getFields("field" + RankVectorsFieldMapper.VECTOR_MAGNITUDES_SUFFIX); assertEquals(1, magFields.size()); assertThat(magFields.get(0), instanceOf(BinaryDocValuesField.class)); BytesRef magBR = magFields.get(0).binaryValue(); @@ -249,7 +249,7 @@ public void testNonIndexedVector() throws Exception { } public void testPoorlyIndexedVector() throws Exception { - DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", 3))); + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "rank_vectors").field("dims", 3))); float[][] validVectors = { { -12.1f, 100.7f, -4 }, { 42f, .05f, -1f } }; double[] dotProduct = new double[2]; @@ -279,27 +279,23 @@ public void testInvalidParameters() { MapperParsingException e = expectThrows( MapperParsingException.class, - () -> createDocumentMapper( - fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", 3).field("element_type", "foo")) - ) + () -> createDocumentMapper(fieldMapping(b -> b.field("type", "rank_vectors").field("dims", 3).field("element_type", "foo"))) ); assertThat(e.getMessage(), containsString("invalid element_type [foo]; available types are ")); e = expectThrows( MapperParsingException.class, - () -> createDocumentMapper( - fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", 3).startObject("foo").endObject()) - ) + () -> createDocumentMapper(fieldMapping(b -> b.field("type", "rank_vectors").field("dims", 3).startObject("foo").endObject())) ); assertThat( e.getMessage(), - containsString("Failed to parse mapping: unknown parameter [foo] on mapper [field] of type [multi_dense_vector]") + containsString("Failed to parse mapping: unknown parameter [foo] on mapper [field] of type [rank_vectors]") ); } public void testDocumentsWithIncorrectDims() throws Exception { int dims = 3; XContentBuilder fieldMapping = fieldMapping(b -> { - b.field("type", "multi_dense_vector"); + b.field("type", "rank_vectors"); b.field("dims", dims); }); @@ -382,8 +378,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); nativeFetcher.setNextReader(ir.leaves().get(0)); List fromNative = nativeFetcher.fetchValues(s, 0, new ArrayList<>()); - MultiDenseVectorFieldMapper.MultiDenseVectorFieldType denseVectorFieldType = - (MultiDenseVectorFieldMapper.MultiDenseVectorFieldType) ft; + RankVectorsFieldMapper.RankVectorsFieldType denseVectorFieldType = (RankVectorsFieldMapper.RankVectorsFieldType) ft; switch (denseVectorFieldType.getElementType()) { case BYTE -> assumeFalse("byte element type testing not currently added", false); case FLOAT -> { @@ -408,12 +403,12 @@ protected void assertFetch(MapperService mapperService, String field, Object val @Override protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { - b.field("type", "multi_dense_vector").field("dims", randomIntBetween(2, 4096)).field("element_type", "float"); + b.field("type", "rank_vectors").field("dims", randomIntBetween(2, 4096)).field("element_type", "float"); } @Override protected Object generateRandomInputValue(MappedFieldType ft) { - MultiDenseVectorFieldMapper.MultiDenseVectorFieldType vectorFieldType = (MultiDenseVectorFieldMapper.MultiDenseVectorFieldType) ft; + RankVectorsFieldMapper.RankVectorsFieldType vectorFieldType = (RankVectorsFieldMapper.RankVectorsFieldType) ft; int numVectors = randomIntBetween(1, 16); return switch (vectorFieldType.getElementType()) { case BYTE -> { @@ -451,7 +446,7 @@ public void testCannotBeUsedInMultifields() { b.endObject(); b.endObject(); }))); - assertThat(e.getMessage(), containsString("Field [vectors] of type [multi_dense_vector] can't be used in multifields")); + assertThat(e.getMessage(), containsString("Field [vectors] of type [rank_vectors] can't be used in multifields")); } @Override @@ -486,7 +481,7 @@ public SyntheticSourceExample example(int maxValues) { } private void mapping(XContentBuilder b) throws IOException { - b.field("type", "multi_dense_vector"); + b.field("type", "rank_vectors"); if (elementType == ElementType.BYTE || elementType == ElementType.BIT || randomBoolean()) { b.field("element_type", elementType.toString()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java similarity index 62% rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldTypeTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java index 14cc63e31fa27..b4cbbc4730d7c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsFieldTypeTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.vectors.MultiDenseVectorFieldMapper.MultiDenseVectorFieldType; +import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper.RankVectorsFieldType; import org.junit.BeforeClass; import java.io.IOException; @@ -23,15 +23,15 @@ import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BBQ_MIN_DIMS; -public class MultiDenseVectorFieldTypeTests extends FieldTypeTestCase { +public class RankVectorsFieldTypeTests extends FieldTypeTestCase { @BeforeClass public static void setup() { - assumeTrue("Requires multi-dense vector support", MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()); + assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); } - private MultiDenseVectorFieldType createFloatFieldType() { - return new MultiDenseVectorFieldType( + private RankVectorsFieldType createFloatFieldType() { + return new RankVectorsFieldType( "f", DenseVectorFieldMapper.ElementType.FLOAT, BBQ_MIN_DIMS, @@ -40,66 +40,60 @@ private MultiDenseVectorFieldType createFloatFieldType() { ); } - private MultiDenseVectorFieldType createByteFieldType() { - return new MultiDenseVectorFieldType( - "f", - DenseVectorFieldMapper.ElementType.BYTE, - 5, - IndexVersion.current(), - Collections.emptyMap() - ); + private RankVectorsFieldType createByteFieldType() { + return new RankVectorsFieldType("f", DenseVectorFieldMapper.ElementType.BYTE, 5, IndexVersion.current(), Collections.emptyMap()); } public void testHasDocValues() { - MultiDenseVectorFieldType fft = createFloatFieldType(); + RankVectorsFieldType fft = createFloatFieldType(); assertTrue(fft.hasDocValues()); - MultiDenseVectorFieldType bft = createByteFieldType(); + RankVectorsFieldType bft = createByteFieldType(); assertTrue(bft.hasDocValues()); } public void testIsIndexed() { - MultiDenseVectorFieldType fft = createFloatFieldType(); + RankVectorsFieldType fft = createFloatFieldType(); assertFalse(fft.isIndexed()); - MultiDenseVectorFieldType bft = createByteFieldType(); + RankVectorsFieldType bft = createByteFieldType(); assertFalse(bft.isIndexed()); } public void testIsSearchable() { - MultiDenseVectorFieldType fft = createFloatFieldType(); + RankVectorsFieldType fft = createFloatFieldType(); assertFalse(fft.isSearchable()); - MultiDenseVectorFieldType bft = createByteFieldType(); + RankVectorsFieldType bft = createByteFieldType(); assertFalse(bft.isSearchable()); } public void testIsAggregatable() { - MultiDenseVectorFieldType fft = createFloatFieldType(); + RankVectorsFieldType fft = createFloatFieldType(); assertFalse(fft.isAggregatable()); - MultiDenseVectorFieldType bft = createByteFieldType(); + RankVectorsFieldType bft = createByteFieldType(); assertFalse(bft.isAggregatable()); } public void testFielddataBuilder() { - MultiDenseVectorFieldType fft = createFloatFieldType(); + RankVectorsFieldType fft = createFloatFieldType(); FieldDataContext fdc = new FieldDataContext("test", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); assertNotNull(fft.fielddataBuilder(fdc)); - MultiDenseVectorFieldType bft = createByteFieldType(); + RankVectorsFieldType bft = createByteFieldType(); FieldDataContext bdc = new FieldDataContext("test", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); assertNotNull(bft.fielddataBuilder(bdc)); } public void testDocValueFormat() { - MultiDenseVectorFieldType fft = createFloatFieldType(); + RankVectorsFieldType fft = createFloatFieldType(); expectThrows(IllegalArgumentException.class, () -> fft.docValueFormat(null, null)); - MultiDenseVectorFieldType bft = createByteFieldType(); + RankVectorsFieldType bft = createByteFieldType(); expectThrows(IllegalArgumentException.class, () -> bft.docValueFormat(null, null)); } public void testFetchSourceValue() throws IOException { - MultiDenseVectorFieldType fft = createFloatFieldType(); + RankVectorsFieldType fft = createFloatFieldType(); List> vector = List.of(List.of(0.0, 1.0, 2.0, 3.0, 4.0, 6.0)); assertEquals(vector, fetchSourceValue(fft, vector)); - MultiDenseVectorFieldType bft = createByteFieldType(); + RankVectorsFieldType bft = createByteFieldType(); assertEquals(vector, fetchSourceValue(bft, vector)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValuesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java similarity index 75% rename from server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValuesTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java index 435baa477e740..c38ed0f60f0ae 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorScriptDocValuesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/RankVectorsScriptDocValuesTests.java @@ -13,10 +13,10 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; -import org.elasticsearch.script.field.vectors.ByteMultiDenseVectorDocValuesField; -import org.elasticsearch.script.field.vectors.FloatMultiDenseVectorDocValuesField; -import org.elasticsearch.script.field.vectors.MultiDenseVector; -import org.elasticsearch.script.field.vectors.MultiDenseVectorDocValuesField; +import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.RankVectors; +import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; import org.elasticsearch.test.ESTestCase; import org.junit.BeforeClass; @@ -27,11 +27,11 @@ import static org.hamcrest.Matchers.containsString; -public class MultiDenseVectorScriptDocValuesTests extends ESTestCase { +public class RankVectorsScriptDocValuesTests extends ESTestCase { @BeforeClass public static void setup() { - assumeTrue("Requires multi-dense vector support", MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()); + assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); } public void testFloatGetVectorValueAndGetMagnitude() throws IOException { @@ -41,14 +41,8 @@ public void testFloatGetVectorValueAndGetMagnitude() throws IOException { BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT); BinaryDocValues magnitudeValues = wrap(expectedMagnitudes); - MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( - docValues, - magnitudeValues, - "test", - ElementType.FLOAT, - dims - ); - MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + RankVectorsDocValuesField field = new FloatRankVectorsDocValuesField(docValues, magnitudeValues, "test", ElementType.FLOAT, dims); + RankVectorsScriptDocValues scriptDocValues = field.toScriptDocValues(); for (int i = 0; i < vectors.length; i++) { field.setNextDocId(i); assertEquals(vectors[i].length, field.size()); @@ -71,14 +65,8 @@ public void testByteGetVectorValueAndGetMagnitude() throws IOException { BinaryDocValues docValues = wrap(vectors, ElementType.BYTE); BinaryDocValues magnitudeValues = wrap(expectedMagnitudes); - MultiDenseVectorDocValuesField field = new ByteMultiDenseVectorDocValuesField( - docValues, - magnitudeValues, - "test", - ElementType.BYTE, - dims - ); - MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + RankVectorsDocValuesField field = new ByteRankVectorsDocValuesField(docValues, magnitudeValues, "test", ElementType.BYTE, dims); + RankVectorsScriptDocValues scriptDocValues = field.toScriptDocValues(); for (int i = 0; i < vectors.length; i++) { field.setNextDocId(i); assertEquals(vectors[i].length, field.size()); @@ -101,25 +89,19 @@ public void testFloatMetadataAndIterator() throws IOException { BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT); BinaryDocValues magnitudeValues = wrap(magnitudes); - MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( - docValues, - magnitudeValues, - "test", - ElementType.FLOAT, - dims - ); + RankVectorsDocValuesField field = new FloatRankVectorsDocValuesField(docValues, magnitudeValues, "test", ElementType.FLOAT, dims); for (int i = 0; i < vectors.length; i++) { field.setNextDocId(i); - MultiDenseVector dv = field.get(); + RankVectors dv = field.get(); assertEquals(vectors[i].length, dv.size()); assertFalse(dv.isEmpty()); assertEquals(dims, dv.getDims()); UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, field::iterator); - assertEquals("Cannot iterate over single valued multi_dense_vector field, use get() instead", e.getMessage()); + assertEquals("Cannot iterate over single valued rank_vectors field, use get() instead", e.getMessage()); } field.setNextDocId(vectors.length); - MultiDenseVector dv = field.get(); - assertEquals(dv, MultiDenseVector.EMPTY); + RankVectors dv = field.get(); + assertEquals(dv, RankVectors.EMPTY); } public void testByteMetadataAndIterator() throws IOException { @@ -128,25 +110,19 @@ public void testByteMetadataAndIterator() throws IOException { float[][] magnitudes = new float[][] { new float[3], new float[2] }; BinaryDocValues docValues = wrap(vectors, ElementType.BYTE); BinaryDocValues magnitudeValues = wrap(magnitudes); - MultiDenseVectorDocValuesField field = new ByteMultiDenseVectorDocValuesField( - docValues, - magnitudeValues, - "test", - ElementType.BYTE, - dims - ); + RankVectorsDocValuesField field = new ByteRankVectorsDocValuesField(docValues, magnitudeValues, "test", ElementType.BYTE, dims); for (int i = 0; i < vectors.length; i++) { field.setNextDocId(i); - MultiDenseVector dv = field.get(); + RankVectors dv = field.get(); assertEquals(vectors[i].length, dv.size()); assertFalse(dv.isEmpty()); assertEquals(dims, dv.getDims()); UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, field::iterator); - assertEquals("Cannot iterate over single valued multi_dense_vector field, use get() instead", e.getMessage()); + assertEquals("Cannot iterate over single valued rank_vectors field, use get() instead", e.getMessage()); } field.setNextDocId(vectors.length); - MultiDenseVector dv = field.get(); - assertEquals(dv, MultiDenseVector.EMPTY); + RankVectors dv = field.get(); + assertEquals(dv, RankVectors.EMPTY); } protected float[][] fill(float[][] vectors, ElementType elementType) { @@ -164,22 +140,16 @@ public void testFloatMissingValues() throws IOException { float[][] magnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT); BinaryDocValues magnitudeValues = wrap(magnitudes); - MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( - docValues, - magnitudeValues, - "test", - ElementType.FLOAT, - dims - ); - MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + RankVectorsDocValuesField field = new FloatRankVectorsDocValuesField(docValues, magnitudeValues, "test", ElementType.FLOAT, dims); + RankVectorsScriptDocValues scriptDocValues = field.toScriptDocValues(); field.setNextDocId(3); assertEquals(0, field.size()); Exception e = expectThrows(IllegalArgumentException.class, scriptDocValues::getVectorValues); - assertEquals("A document doesn't have a value for a multi-vector field!", e.getMessage()); + assertEquals("A document doesn't have a value for a rank-vectors field!", e.getMessage()); e = expectThrows(IllegalArgumentException.class, scriptDocValues::getMagnitudes); - assertEquals("A document doesn't have a value for a multi-vector field!", e.getMessage()); + assertEquals("A document doesn't have a value for a rank-vectors field!", e.getMessage()); } public void testByteMissingValues() throws IOException { @@ -188,22 +158,16 @@ public void testByteMissingValues() throws IOException { float[][] magnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; BinaryDocValues docValues = wrap(vectors, ElementType.BYTE); BinaryDocValues magnitudeValues = wrap(magnitudes); - MultiDenseVectorDocValuesField field = new ByteMultiDenseVectorDocValuesField( - docValues, - magnitudeValues, - "test", - ElementType.BYTE, - dims - ); - MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + RankVectorsDocValuesField field = new ByteRankVectorsDocValuesField(docValues, magnitudeValues, "test", ElementType.BYTE, dims); + RankVectorsScriptDocValues scriptDocValues = field.toScriptDocValues(); field.setNextDocId(3); assertEquals(0, field.size()); Exception e = expectThrows(IllegalArgumentException.class, scriptDocValues::getVectorValues); - assertEquals("A document doesn't have a value for a multi-vector field!", e.getMessage()); + assertEquals("A document doesn't have a value for a rank-vectors field!", e.getMessage()); e = expectThrows(IllegalArgumentException.class, scriptDocValues::getMagnitudes); - assertEquals("A document doesn't have a value for a multi-vector field!", e.getMessage()); + assertEquals("A document doesn't have a value for a rank-vectors field!", e.getMessage()); } public void testFloatGetFunctionIsNotAccessible() throws IOException { @@ -212,21 +176,15 @@ public void testFloatGetFunctionIsNotAccessible() throws IOException { float[][] magnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; BinaryDocValues docValues = wrap(vectors, ElementType.FLOAT); BinaryDocValues magnitudeValues = wrap(magnitudes); - MultiDenseVectorDocValuesField field = new FloatMultiDenseVectorDocValuesField( - docValues, - magnitudeValues, - "test", - ElementType.FLOAT, - dims - ); - MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + RankVectorsDocValuesField field = new FloatRankVectorsDocValuesField(docValues, magnitudeValues, "test", ElementType.FLOAT, dims); + RankVectorsScriptDocValues scriptDocValues = field.toScriptDocValues(); field.setNextDocId(0); Exception e = expectThrows(UnsupportedOperationException.class, () -> scriptDocValues.get(0)); assertThat( e.getMessage(), containsString( - "accessing a multi-vector field's value through 'get' or 'value' is not supported," + "accessing a rank-vectors field's value through 'get' or 'value' is not supported," + " use 'vectorValues' or 'magnitudes' instead." ) ); @@ -238,21 +196,15 @@ public void testByteGetFunctionIsNotAccessible() throws IOException { float[][] magnitudes = { { 1.7320f, 2.4495f, 3.3166f }, { 2.2361f } }; BinaryDocValues docValues = wrap(vectors, ElementType.BYTE); BinaryDocValues magnitudeValues = wrap(magnitudes); - MultiDenseVectorDocValuesField field = new ByteMultiDenseVectorDocValuesField( - docValues, - magnitudeValues, - "test", - ElementType.BYTE, - dims - ); - MultiDenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues(); + RankVectorsDocValuesField field = new ByteRankVectorsDocValuesField(docValues, magnitudeValues, "test", ElementType.BYTE, dims); + RankVectorsScriptDocValues scriptDocValues = field.toScriptDocValues(); field.setNextDocId(0); Exception e = expectThrows(UnsupportedOperationException.class, () -> scriptDocValues.get(0)); assertThat( e.getMessage(), containsString( - "accessing a multi-vector field's value through 'get' or 'value' is not supported," + "accessing a rank-vectors field's value through 'get' or 'value' is not supported," + " use 'vectorValues' or 'magnitudes' instead." ) ); diff --git a/server/src/test/java/org/elasticsearch/script/MultiVectorScoreScriptUtilsTests.java b/server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java similarity index 81% rename from server/src/test/java/org/elasticsearch/script/MultiVectorScoreScriptUtilsTests.java rename to server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java index f908f51170478..917cc2069a293 100644 --- a/server/src/test/java/org/elasticsearch/script/MultiVectorScoreScriptUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/script/RankVectorsScoreScriptUtilsTests.java @@ -11,14 +11,14 @@ import org.apache.lucene.util.VectorUtil; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; -import org.elasticsearch.index.mapper.vectors.MultiDenseVectorFieldMapper; -import org.elasticsearch.index.mapper.vectors.MultiDenseVectorScriptDocValuesTests; -import org.elasticsearch.script.MultiVectorScoreScriptUtils.MaxSimDotProduct; -import org.elasticsearch.script.MultiVectorScoreScriptUtils.MaxSimInvHamming; -import org.elasticsearch.script.field.vectors.BitMultiDenseVectorDocValuesField; -import org.elasticsearch.script.field.vectors.ByteMultiDenseVectorDocValuesField; -import org.elasticsearch.script.field.vectors.FloatMultiDenseVectorDocValuesField; -import org.elasticsearch.script.field.vectors.MultiDenseVectorDocValuesField; +import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; +import org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValuesTests; +import org.elasticsearch.script.RankVectorsScoreScriptUtils.MaxSimDotProduct; +import org.elasticsearch.script.RankVectorsScoreScriptUtils.MaxSimInvHamming; +import org.elasticsearch.script.field.vectors.BitRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.ByteRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.FloatRankVectorsDocValuesField; +import org.elasticsearch.script.field.vectors.RankVectorsDocValuesField; import org.elasticsearch.test.ESTestCase; import org.junit.BeforeClass; @@ -31,11 +31,11 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class MultiVectorScoreScriptUtilsTests extends ESTestCase { +public class RankVectorsScoreScriptUtilsTests extends ESTestCase { @BeforeClass public static void setup() { - assumeTrue("Requires multi-dense vector support", MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()); + assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); } public void testFloatMultiVectorClassBindings() throws IOException { @@ -53,23 +53,23 @@ public void testFloatMultiVectorClassBindings() throws IOException { List> queryVector = List.of(Arrays.asList(0.5f, 111.3f, -13.0f, 14.8f, -156.0f)); List> invalidQueryVector = List.of(Arrays.asList(0.5, 111.3)); - List fields = List.of( - new FloatMultiDenseVectorDocValuesField( - MultiDenseVectorScriptDocValuesTests.wrap(docVectors, ElementType.FLOAT), - MultiDenseVectorScriptDocValuesTests.wrap(docMagnitudes), + List fields = List.of( + new FloatRankVectorsDocValuesField( + RankVectorsScriptDocValuesTests.wrap(docVectors, ElementType.FLOAT), + RankVectorsScriptDocValuesTests.wrap(docMagnitudes), "test", ElementType.FLOAT, dims ), - new FloatMultiDenseVectorDocValuesField( - MultiDenseVectorScriptDocValuesTests.wrap(docVectors, ElementType.FLOAT), - MultiDenseVectorScriptDocValuesTests.wrap(docMagnitudes), + new FloatRankVectorsDocValuesField( + RankVectorsScriptDocValuesTests.wrap(docVectors, ElementType.FLOAT), + RankVectorsScriptDocValuesTests.wrap(docMagnitudes), "test", ElementType.FLOAT, dims ) ); - for (MultiDenseVectorDocValuesField field : fields) { + for (RankVectorsDocValuesField field : fields) { field.setNextDocId(0); ScoreScript scoreScript = mock(ScoreScript.class); @@ -88,7 +88,7 @@ public void testFloatMultiVectorClassBindings() throws IOException { // Check each function rejects query vectors with the wrong dimension IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new MultiVectorScoreScriptUtils.MaxSimDotProduct(scoreScript, invalidQueryVector, fieldName) + () -> new RankVectorsScoreScriptUtils.MaxSimDotProduct(scoreScript, invalidQueryVector, fieldName) ); assertThat( e.getMessage(), @@ -120,16 +120,16 @@ public void testByteMultiVectorClassBindings() throws IOException { List> invalidQueryVector = List.of(Arrays.asList((byte) 1, (byte) 1)); List hexidecimalString = List.of(HexFormat.of().formatHex(new byte[] { 1, 125, -12, 2, 4 })); - List fields = List.of( - new ByteMultiDenseVectorDocValuesField( - MultiDenseVectorScriptDocValuesTests.wrap(new float[][][] { docVector }, ElementType.BYTE), - MultiDenseVectorScriptDocValuesTests.wrap(magnitudes), + List fields = List.of( + new ByteRankVectorsDocValuesField( + RankVectorsScriptDocValuesTests.wrap(new float[][][] { docVector }, ElementType.BYTE), + RankVectorsScriptDocValuesTests.wrap(magnitudes), "test", ElementType.BYTE, dims ) ); - for (MultiDenseVectorDocValuesField field : fields) { + for (RankVectorsDocValuesField field : fields) { field.setNextDocId(0); ScoreScript scoreScript = mock(ScoreScript.class); @@ -174,16 +174,16 @@ public void testBitMultiVectorClassBindingsDotProduct() throws IOException { List> invalidQueryVector = List.of(Arrays.asList((byte) 1, (byte) 1)); List hexidecimalString = List.of(HexFormat.of().formatHex(new byte[] { 124 })); - List fields = List.of( - new BitMultiDenseVectorDocValuesField( - MultiDenseVectorScriptDocValuesTests.wrap(new float[][][] { docVector }, ElementType.BIT), - MultiDenseVectorScriptDocValuesTests.wrap(new float[][] { { 5 } }), + List fields = List.of( + new BitRankVectorsDocValuesField( + RankVectorsScriptDocValuesTests.wrap(new float[][][] { docVector }, ElementType.BIT), + RankVectorsScriptDocValuesTests.wrap(new float[][] { { 5 } }), "test", ElementType.BIT, dims ) ); - for (MultiDenseVectorDocValuesField field : fields) { + for (RankVectorsDocValuesField field : fields) { field.setNextDocId(0); ScoreScript scoreScript = mock(ScoreScript.class); @@ -240,23 +240,23 @@ public void testByteVsFloatSimilarity() throws IOException { float[][] floatVector = new float[][] { { 1f, 125f, -12f, 2f, 4f } }; byte[][] byteVector = new byte[][] { { (byte) 1, (byte) 125, (byte) -12, (byte) 2, (byte) 4 } }; - List fields = List.of( - new FloatMultiDenseVectorDocValuesField( - MultiDenseVectorScriptDocValuesTests.wrap(new float[][][] { docVector }, ElementType.FLOAT), - MultiDenseVectorScriptDocValuesTests.wrap(magnitudes), + List fields = List.of( + new FloatRankVectorsDocValuesField( + RankVectorsScriptDocValuesTests.wrap(new float[][][] { docVector }, ElementType.FLOAT), + RankVectorsScriptDocValuesTests.wrap(magnitudes), "field1", ElementType.FLOAT, dims ), - new ByteMultiDenseVectorDocValuesField( - MultiDenseVectorScriptDocValuesTests.wrap(new float[][][] { docVector }, ElementType.BYTE), - MultiDenseVectorScriptDocValuesTests.wrap(magnitudes), + new ByteRankVectorsDocValuesField( + RankVectorsScriptDocValuesTests.wrap(new float[][][] { docVector }, ElementType.BYTE), + RankVectorsScriptDocValuesTests.wrap(magnitudes), "field3", ElementType.BYTE, dims ) ); - for (MultiDenseVectorDocValuesField field : fields) { + for (RankVectorsDocValuesField field : fields) { field.setNextDocId(0); ScoreScript scoreScript = mock(ScoreScript.class); @@ -296,17 +296,17 @@ public void testByteBoundaries() throws IOException { List> lessThanVector = List.of(List.of(-129)); List> decimalVector = List.of(List.of(0.5)); - List fields = List.of( - new ByteMultiDenseVectorDocValuesField( - MultiDenseVectorScriptDocValuesTests.wrap(new float[][][] { { docVector } }, ElementType.BYTE), - MultiDenseVectorScriptDocValuesTests.wrap(new float[][] { { 1 } }), + List fields = List.of( + new ByteRankVectorsDocValuesField( + RankVectorsScriptDocValuesTests.wrap(new float[][][] { { docVector } }, ElementType.BYTE), + RankVectorsScriptDocValuesTests.wrap(new float[][] { { 1 } }), "test", ElementType.BYTE, dims ) ); - for (MultiDenseVectorDocValuesField field : fields) { + for (RankVectorsDocValuesField field : fields) { field.setNextDocId(0); ScoreScript scoreScript = mock(ScoreScript.class); diff --git a/server/src/test/java/org/elasticsearch/script/field/vectors/MultiDenseVectorTests.java b/server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java similarity index 80% rename from server/src/test/java/org/elasticsearch/script/field/vectors/MultiDenseVectorTests.java rename to server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java index 12f4b931b4d0a..ca7608b10aed9 100644 --- a/server/src/test/java/org/elasticsearch/script/field/vectors/MultiDenseVectorTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/vectors/RankVectorsTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.VectorUtil; -import org.elasticsearch.index.mapper.vectors.MultiDenseVectorFieldMapper; +import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import org.elasticsearch.test.ESTestCase; import org.junit.BeforeClass; @@ -19,11 +19,11 @@ import java.nio.ByteOrder; import java.util.function.IntFunction; -public class MultiDenseVectorTests extends ESTestCase { +public class RankVectorsTests extends ESTestCase { @BeforeClass public static void setup() { - assumeTrue("Requires multi-dense vector support", MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()); + assumeTrue("Requires rank-vectors support", RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()); } public void testByteUnsupported() { @@ -38,7 +38,7 @@ public void testByteUnsupported() { } } - MultiDenseVector knn = newByteVector(docVector); + RankVectors knn = newByteVector(docVector); UnsupportedOperationException e; e = expectThrows(UnsupportedOperationException.class, () -> knn.maxSimDotProduct(queryVector)); @@ -57,20 +57,20 @@ public void testFloatUnsupported() { } } - MultiDenseVector knn = newFloatVector(docVector); + RankVectors knn = newFloatVector(docVector); UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, () -> knn.maxSimDotProduct(queryVector)); assertEquals(e.getMessage(), "use [float maxSimDotProduct(float[][] queryVector)] instead"); } - static MultiDenseVector newFloatVector(float[][] vector) { + static RankVectors newFloatVector(float[][] vector) { BytesRef magnitudes = magnitudes(vector.length, i -> (float) Math.sqrt(VectorUtil.dotProduct(vector[i], vector[i]))); - return new FloatMultiDenseVector(VectorIterator.from(vector), magnitudes, vector.length, vector[0].length); + return new FloatRankVectors(VectorIterator.from(vector), magnitudes, vector.length, vector[0].length); } - static MultiDenseVector newByteVector(byte[][] vector) { + static RankVectors newByteVector(byte[][] vector) { BytesRef magnitudes = magnitudes(vector.length, i -> (float) Math.sqrt(VectorUtil.dotProduct(vector[i], vector[i]))); - return new ByteMultiDenseVector(VectorIterator.from(vector), magnitudes, vector.length, vector[0].length); + return new ByteRankVectors(VectorIterator.from(vector), magnitudes, vector.length, vector[0].length); } static BytesRef magnitudes(int count, IntFunction magnitude) { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 51f66418bb44b..d491e4bb52fa2 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -112,7 +112,7 @@ import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; -import org.elasticsearch.index.mapper.vectors.MultiDenseVectorFieldMapper; +import org.elasticsearch.index.mapper.vectors.RankVectorsFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; @@ -206,7 +206,7 @@ public abstract class AggregatorTestCase extends ESTestCase { private static final List TYPE_TEST_BLACKLIST = List.of( ObjectMapper.CONTENT_TYPE, // Cannot aggregate objects DenseVectorFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors - MultiDenseVectorFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors + RankVectorsFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors SparseVectorFieldMapper.CONTENT_TYPE, // Sparse vectors are no longer supported NestedObjectMapper.CONTENT_TYPE, // TODO support for nested From 98e69c890cfa9b61c201d0a77dc5e3e9560b3fb5 Mon Sep 17 00:00:00 2001 From: Dan Rubinstein Date: Mon, 9 Dec 2024 13:43:06 -0500 Subject: [PATCH 137/204] Adding deprecation warning for data_frame_transforms roles (#117521) * Adding deprecation warning for data_frame_transforms roles * Updating deprecation warning URL --------- Co-authored-by: Elastic Machine --- .../core/transform/TransformDeprecations.java | 7 ++ .../transform/transforms/TransformConfig.java | 39 +++++++++- .../transforms/TransformConfigTests.java | 77 ++++++++++++++++++- 3 files changed, 119 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java index 79a679441de3a..1de584d5593f1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java @@ -27,5 +27,12 @@ public class TransformDeprecations { public static final String MAX_PAGE_SEARCH_SIZE_BREAKING_CHANGES_URL = "https://ela.st/es-deprecation-7-transform-max-page-search-size"; + public static final String DATA_FRAME_TRANSFORMS_ROLES_BREAKING_CHANGES_URL = + "https://ela.st/es-deprecation-9-data-frame-transforms-roles"; + + public static final String DATA_FRAME_TRANSFORMS_ROLES_IS_DEPRECATED = "This transform configuration uses one or more obsolete roles " + + "prefixed with [data_frame_transformers_] which will be unsupported after the next upgrade. Switch to a user with the equivalent " + + "roles prefixed with [transform_] and use [/_transform/_upgrade] to upgrade all transforms to the latest roles.";; + private TransformDeprecations() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index d8972dcf6a6be..745da71539992 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -24,11 +24,13 @@ import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.common.time.TimeUtils; import org.elasticsearch.xpack.core.common.validation.SourceDestValidator; import org.elasticsearch.xpack.core.common.validation.SourceDestValidator.SourceDestValidation; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level; +import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; import org.elasticsearch.xpack.core.security.xcontent.XContentUtils; import org.elasticsearch.xpack.core.transform.TransformConfigVersion; import org.elasticsearch.xpack.core.transform.TransformDeprecations; @@ -41,6 +43,7 @@ import java.io.IOException; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -49,6 +52,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; /** * This class holds the configuration details of a data frame transform @@ -65,6 +69,10 @@ public final class TransformConfig implements SimpleDiffable, W public static final ParseField HEADERS = new ParseField("headers"); /** Version in which {@code FieldCapabilitiesRequest.runtime_fields} field was introduced. */ private static final TransportVersion FIELD_CAPS_RUNTIME_MAPPINGS_INTRODUCED_TRANSPORT_VERSION = TransportVersions.V_7_12_0; + private static final List DEPRECATED_DATA_FRAME_TRANSFORMS_ROLES = List.of( + "data_frame_transforms_admin", + "data_frame_transforms_user" + ); /** Specifies all the possible transform functions. */ public enum Function { @@ -374,7 +382,7 @@ public ActionRequestValidationException validate(ActionRequestValidationExceptio * @param namedXContentRegistry XContent registry required for aggregations and query DSL * @return The deprecations of this transform */ - public List checkForDeprecations(NamedXContentRegistry namedXContentRegistry) { + public List checkForDeprecations(NamedXContentRegistry namedXContentRegistry) throws IOException { List deprecations = new ArrayList<>(); @@ -404,9 +412,38 @@ public List checkForDeprecations(NamedXContentRegistry namedXC if (retentionPolicyConfig != null) { retentionPolicyConfig.checkForDeprecations(getId(), namedXContentRegistry, deprecations::add); } + + var deprecatedTransformRoles = getRolesFromHeaders().stream().filter(DEPRECATED_DATA_FRAME_TRANSFORMS_ROLES::contains).toList(); + if (deprecatedTransformRoles.isEmpty() == false) { + deprecations.add( + new DeprecationIssue( + Level.CRITICAL, + "Transform [" + id + "] uses deprecated transform roles " + deprecatedTransformRoles, + TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_BREAKING_CHANGES_URL, + TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_IS_DEPRECATED, + false, + null + ) + ); + } + return deprecations; } + private List getRolesFromHeaders() throws IOException { + if (headers == null) { + return Collections.emptyList(); + } + + var encodedAuthenticationHeader = ClientHelper.filterSecurityHeaders(headers).getOrDefault(AUTHENTICATION_KEY, ""); + if (encodedAuthenticationHeader.isEmpty()) { + return Collections.emptyList(); + } + + var decodedAuthenticationHeader = AuthenticationContextSerializer.decode(encodedAuthenticationHeader); + return Arrays.asList(decodedAuthenticationHeader.getEffectiveSubject().getUser().roles()); + } + @Override public void writeTo(final StreamOutput out) throws IOException { out.writeString(id); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index 8cfecc432c661..2e7e5293c835f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -27,6 +27,8 @@ import org.elasticsearch.xpack.core.common.validation.SourceDestValidator.SourceDestValidation; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level; +import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; +import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.elasticsearch.xpack.core.transform.TransformConfigVersion; import org.elasticsearch.xpack.core.transform.TransformDeprecations; @@ -44,6 +46,7 @@ import java.util.Map; import static org.elasticsearch.test.TestMatchers.matchesPattern; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; import static org.elasticsearch.xpack.core.transform.transforms.DestConfigTests.randomDestConfig; import static org.elasticsearch.xpack.core.transform.transforms.SourceConfigTests.randomInvalidSourceConfig; import static org.elasticsearch.xpack.core.transform.transforms.SourceConfigTests.randomSourceConfig; @@ -58,6 +61,8 @@ public class TransformConfigTests extends AbstractSerializingTransformTestCase headers) { + return new TransformConfig( + randomAlphaOfLengthBetween(1, 10), + randomSourceConfig(), + randomDestConfig(), + randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), + randomBoolean() ? null : randomSyncConfig(), + headers, + randomBoolean() ? null : PivotConfigTests.randomPivotConfig(), + randomBoolean() ? null : LatestConfigTests.randomLatestConfig(), + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), + randomBoolean() ? null : SettingsConfigTests.randomSettingsConfig(), + randomBoolean() ? null : randomMetadata(), + randomBoolean() ? null : randomRetentionPolicyConfig(), + randomBoolean() ? null : Instant.now(), + TransformConfigVersion.CURRENT.toString() + ); + } + public static TransformConfig randomTransformConfig( String id, TransformConfigVersion version, @@ -915,10 +939,13 @@ public void testGroupByStayInOrder() throws IOException { } } - public void testCheckForDeprecations() { + public void testCheckForDeprecations_NoDeprecationWarnings() throws IOException { String id = randomAlphaOfLengthBetween(1, 10); assertThat(randomTransformConfig(id, TransformConfigVersion.CURRENT).checkForDeprecations(xContentRegistry()), is(empty())); + } + public void testCheckForDeprecations_WithDeprecatedFields_VersionCurrent() throws IOException { + String id = randomAlphaOfLengthBetween(1, 10); TransformConfig deprecatedConfig = randomTransformConfigWithDeprecatedFields(id, TransformConfigVersion.CURRENT); // check _and_ clear warnings @@ -940,8 +967,11 @@ public void testCheckForDeprecations() { ) ) ); + } - deprecatedConfig = randomTransformConfigWithDeprecatedFields(id, TransformConfigVersion.V_7_10_0); + public void testCheckForDeprecations_WithDeprecatedFields_Version_7_10() throws IOException { + String id = randomAlphaOfLengthBetween(1, 10); + TransformConfig deprecatedConfig = randomTransformConfigWithDeprecatedFields(id, TransformConfigVersion.V_7_10_0); // check _and_ clear warnings assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED); @@ -962,8 +992,11 @@ public void testCheckForDeprecations() { ) ) ); + } - deprecatedConfig = randomTransformConfigWithDeprecatedFields(id, TransformConfigVersion.V_7_4_0); + public void testCheckForDeprecations_WithDeprecatedFields_Version_7_4() throws IOException { + String id = randomAlphaOfLengthBetween(1, 10); + TransformConfig deprecatedConfig = randomTransformConfigWithDeprecatedFields(id, TransformConfigVersion.V_7_4_0); // check _and_ clear warnings assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED); @@ -994,6 +1027,44 @@ public void testCheckForDeprecations() { ); } + public void testCheckForDeprecations_WithDeprecatedTransformUserAdmin() throws IOException { + testCheckForDeprecations_WithDeprecatedRoles(List.of(DATA_FRAME_TRANSFORMS_ADMIN_ROLE)); + } + + public void testCheckForDeprecations_WithDeprecatedTransformUserRole() throws IOException { + testCheckForDeprecations_WithDeprecatedRoles(List.of(DATA_FRAME_TRANSFORMS_USER_ROLE)); + } + + public void testCheckForDeprecations_WithDeprecatedTransformRoles() throws IOException { + testCheckForDeprecations_WithDeprecatedRoles(List.of(DATA_FRAME_TRANSFORMS_ADMIN_ROLE, DATA_FRAME_TRANSFORMS_USER_ROLE)); + } + + private void testCheckForDeprecations_WithDeprecatedRoles(List roles) throws IOException { + var authentication = AuthenticationTestHelper.builder() + .realm() + .user(new User(randomAlphaOfLength(10), roles.toArray(String[]::new))) + .build(); + Map headers = Map.of(AUTHENTICATION_KEY, authentication.encode()); + TransformConfig deprecatedConfig = randomTransformConfigWithHeaders(headers); + + // important: checkForDeprecations does _not_ create new deprecation warnings + assertThat( + deprecatedConfig.checkForDeprecations(xContentRegistry()), + equalTo( + List.of( + new DeprecationIssue( + Level.CRITICAL, + "Transform [" + deprecatedConfig.getId() + "] uses deprecated transform roles " + roles, + TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_BREAKING_CHANGES_URL, + TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_IS_DEPRECATED, + false, + null + ) + ) + ) + ); + } + public void testSerializingMetadataPreservesOrder() throws IOException { String json = Strings.format(""" { From eb59b989efcb47b181cd4acf83689f97368152a4 Mon Sep 17 00:00:00 2001 From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> Date: Mon, 9 Dec 2024 19:56:10 +0100 Subject: [PATCH 138/204] ESQL: Expand type compatibility for match function and operator (#117555) --- docs/changelog/117555.yaml | 5 + .../esql/functions/description/match.asciidoc | 2 +- .../esql/functions/description/qstr.asciidoc | 2 +- .../functions/kibana/definition/match.json | 466 +++++- .../kibana/definition/match_operator.json | 466 +++++- .../functions/kibana/definition/qstr.json | 2 +- .../esql/functions/kibana/docs/match.md | 2 +- .../functions/kibana/docs/match_operator.md | 2 +- .../esql/functions/kibana/docs/qstr.md | 2 +- .../esql/functions/parameters/match.asciidoc | 2 +- .../esql/functions/search-functions.asciidoc | 8 + docs/reference/esql/functions/search.asciidoc | 5 +- .../esql/functions/types/match.asciidoc | 29 +- .../functions/types/match_operator.asciidoc | 29 +- x-pack/plugin/build.gradle | 3 +- .../xpack/esql/CsvTestsDataLoader.java | 6 + .../xpack/esql/EsqlTestUtils.java | 4 +- .../elasticsearch/xpack/esql/SpecReader.java | 2 +- .../main/resources/employees_incompatible.csv | 101 ++ .../src/main/resources/mapping-all-types.json | 3 + .../src/main/resources/mapping-basic.json | 3 + .../mapping-default-incompatible.json | 80 + .../main/resources/match-function.csv-spec | 326 ++++ .../main/resources/match-operator.csv-spec | 316 ++++ .../xpack/esql/plugin/MatchFunctionIT.java | 10 - .../xpack/esql/plugin/MatchOperatorIT.java | 13 +- .../esql/src/main/antlr/EsqlBaseParser.g4 | 2 +- .../xpack/esql/action/EsqlCapabilities.java | 7 +- .../function/fulltext/FullTextFunction.java | 24 +- .../expression/function/fulltext/Match.java | 134 +- .../function/fulltext/QueryString.java | 3 +- .../comparison/EsqlBinaryComparison.java | 26 +- .../physical/local/PushFiltersToSource.java | 3 - .../xpack/esql/parser/EsqlBaseParser.interp | 2 +- .../xpack/esql/parser/EsqlBaseParser.java | 1408 +++++++++-------- .../xpack/esql/parser/ExpressionBuilder.java | 19 +- .../planner/EsqlExpressionTranslators.java | 25 +- .../xpack/esql/analysis/AnalyzerTests.java | 218 ++- .../xpack/esql/analysis/VerifierTests.java | 5 - .../function/AbstractFunctionTestCase.java | 5 +- .../function/fulltext/MatchOperatorTests.java | 13 +- .../function/fulltext/MatchTests.java | 426 ++++- .../LocalLogicalPlanOptimizerTests.java | 1 + .../LocalPhysicalPlanOptimizerTests.java | 228 ++- .../optimizer/PhysicalPlanOptimizerTests.java | 16 +- .../esql/parser/StatementParserTests.java | 24 +- .../test/esql/180_match_operator.yml | 34 +- 47 files changed, 3557 insertions(+), 955 deletions(-) create mode 100644 docs/changelog/117555.yaml create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees_incompatible.csv create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-default-incompatible.json diff --git a/docs/changelog/117555.yaml b/docs/changelog/117555.yaml new file mode 100644 index 0000000000000..7891ab6d93a64 --- /dev/null +++ b/docs/changelog/117555.yaml @@ -0,0 +1,5 @@ +pr: 117555 +summary: Expand type compatibility for match function and operator +area: ES|QL +type: feature +issues: [] diff --git a/docs/reference/esql/functions/description/match.asciidoc b/docs/reference/esql/functions/description/match.asciidoc index 2a27fe4814395..25f0571878d47 100644 --- a/docs/reference/esql/functions/description/match.asciidoc +++ b/docs/reference/esql/functions/description/match.asciidoc @@ -2,4 +2,4 @@ *Description* -Performs a match query on the specified field. Returns true if the provided query matches the row. +Performs a <> on the specified field. Returns true if the provided query matches the row. diff --git a/docs/reference/esql/functions/description/qstr.asciidoc b/docs/reference/esql/functions/description/qstr.asciidoc index 5ce9316405ad2..d9dbe364f607a 100644 --- a/docs/reference/esql/functions/description/qstr.asciidoc +++ b/docs/reference/esql/functions/description/qstr.asciidoc @@ -2,4 +2,4 @@ *Description* -Performs a query string query. Returns true if the provided query string matches the row. +Performs a <>. Returns true if the provided query string matches the row. diff --git a/docs/reference/esql/functions/kibana/definition/match.json b/docs/reference/esql/functions/kibana/definition/match.json index 4a5b05a3f501b..7f2a8239cc0d0 100644 --- a/docs/reference/esql/functions/kibana/definition/match.json +++ b/docs/reference/esql/functions/kibana/definition/match.json @@ -2,21 +2,75 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "match", - "description" : "Performs a match query on the specified field. Returns true if the provided query matches the row.", + "description" : "Performs a <> on the specified field. Returns true if the provided query matches the row.", "signatures" : [ { "params" : [ { "name" : "field", + "type" : "boolean", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "boolean", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "boolean", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", "type" : "keyword", "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "date", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date", + "optional" : false, "description" : "Field that the query will target." }, { "name" : "query", "type" : "keyword", "optional" : false, - "description" : "Text you wish to find in the provided field." + "description" : "Value to find in the provided field." } ], "variadic" : false, @@ -26,15 +80,51 @@ "params" : [ { "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "date_nanos", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", "type" : "keyword", "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, "description" : "Field that the query will target." }, { "name" : "query", - "type" : "text", + "type" : "double", "optional" : false, - "description" : "Text you wish to find in the provided field." + "description" : "Value to find in the provided field." } ], "variadic" : false, @@ -44,7 +134,25 @@ "params" : [ { "name" : "field", - "type" : "text", + "type" : "double", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", "optional" : false, "description" : "Field that the query will target." }, @@ -52,7 +160,7 @@ "name" : "query", "type" : "keyword", "optional" : false, - "description" : "Text you wish to find in the provided field." + "description" : "Value to find in the provided field." } ], "variadic" : false, @@ -62,15 +170,357 @@ "params" : [ { "name" : "field", - "type" : "text", + "type" : "double", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "ip", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "ip", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "ip", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", "optional" : false, "description" : "Field that the query will target." }, { "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", "type" : "text", "optional" : false, - "description" : "Text you wish to find in the provided field." + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "unsigned_long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "version", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "version", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "version", + "optional" : false, + "description" : "Value to find in the provided field." } ], "variadic" : false, diff --git a/docs/reference/esql/functions/kibana/definition/match_operator.json b/docs/reference/esql/functions/kibana/definition/match_operator.json index 7a0ace6168b59..44233bbddb653 100644 --- a/docs/reference/esql/functions/kibana/definition/match_operator.json +++ b/docs/reference/esql/functions/kibana/definition/match_operator.json @@ -2,21 +2,75 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "operator", "name" : "match_operator", - "description" : "Performs a match query on the specified field. Returns true if the provided query matches the row.", + "description" : "Performs a <> on the specified field. Returns true if the provided query matches the row.", "signatures" : [ { "params" : [ { "name" : "field", + "type" : "boolean", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "boolean", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "boolean", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", "type" : "keyword", "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "date", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date", + "optional" : false, "description" : "Field that the query will target." }, { "name" : "query", "type" : "keyword", "optional" : false, - "description" : "Text you wish to find in the provided field." + "description" : "Value to find in the provided field." } ], "variadic" : false, @@ -26,15 +80,51 @@ "params" : [ { "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "date_nanos", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", "type" : "keyword", "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, "description" : "Field that the query will target." }, { "name" : "query", - "type" : "text", + "type" : "double", "optional" : false, - "description" : "Text you wish to find in the provided field." + "description" : "Value to find in the provided field." } ], "variadic" : false, @@ -44,7 +134,25 @@ "params" : [ { "name" : "field", - "type" : "text", + "type" : "double", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", "optional" : false, "description" : "Field that the query will target." }, @@ -52,7 +160,7 @@ "name" : "query", "type" : "keyword", "optional" : false, - "description" : "Text you wish to find in the provided field." + "description" : "Value to find in the provided field." } ], "variadic" : false, @@ -62,15 +170,357 @@ "params" : [ { "name" : "field", - "type" : "text", + "type" : "double", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "ip", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "ip", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "ip", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", "optional" : false, "description" : "Field that the query will target." }, { "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", "type" : "text", "optional" : false, - "description" : "Text you wish to find in the provided field." + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "unsigned_long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "version", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "version", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "version", + "optional" : false, + "description" : "Value to find in the provided field." } ], "variadic" : false, diff --git a/docs/reference/esql/functions/kibana/definition/qstr.json b/docs/reference/esql/functions/kibana/definition/qstr.json index 76473349a3414..3b091bfe2e13b 100644 --- a/docs/reference/esql/functions/kibana/definition/qstr.json +++ b/docs/reference/esql/functions/kibana/definition/qstr.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "qstr", - "description" : "Performs a query string query. Returns true if the provided query string matches the row.", + "description" : "Performs a <>. Returns true if the provided query string matches the row.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/docs/match.md b/docs/reference/esql/functions/kibana/docs/match.md index b866637b41b85..adf6de91c90f1 100644 --- a/docs/reference/esql/functions/kibana/docs/match.md +++ b/docs/reference/esql/functions/kibana/docs/match.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MATCH -Performs a match query on the specified field. Returns true if the provided query matches the row. +Performs a <> on the specified field. Returns true if the provided query matches the row. ``` FROM books diff --git a/docs/reference/esql/functions/kibana/docs/match_operator.md b/docs/reference/esql/functions/kibana/docs/match_operator.md index fda8b24ff76cc..b0b6196798087 100644 --- a/docs/reference/esql/functions/kibana/docs/match_operator.md +++ b/docs/reference/esql/functions/kibana/docs/match_operator.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MATCH_OPERATOR -Performs a match query on the specified field. Returns true if the provided query matches the row. +Performs a <> on the specified field. Returns true if the provided query matches the row. ``` FROM books diff --git a/docs/reference/esql/functions/kibana/docs/qstr.md b/docs/reference/esql/functions/kibana/docs/qstr.md index 9b5dc3f9a22eb..7df5a2fe08a9d 100644 --- a/docs/reference/esql/functions/kibana/docs/qstr.md +++ b/docs/reference/esql/functions/kibana/docs/qstr.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### QSTR -Performs a query string query. Returns true if the provided query string matches the row. +Performs a <>. Returns true if the provided query string matches the row. ``` FROM books diff --git a/docs/reference/esql/functions/parameters/match.asciidoc b/docs/reference/esql/functions/parameters/match.asciidoc index f18adb28cd20c..46f6acad9e128 100644 --- a/docs/reference/esql/functions/parameters/match.asciidoc +++ b/docs/reference/esql/functions/parameters/match.asciidoc @@ -6,4 +6,4 @@ Field that the query will target. `query`:: -Text you wish to find in the provided field. +Value to find in the provided field. diff --git a/docs/reference/esql/functions/search-functions.asciidoc b/docs/reference/esql/functions/search-functions.asciidoc index 943a262497d4c..238813c382c8c 100644 --- a/docs/reference/esql/functions/search-functions.asciidoc +++ b/docs/reference/esql/functions/search-functions.asciidoc @@ -5,6 +5,14 @@ Full-text Search functions ++++ +Full text functions are used to search for text in fields. +<> is used to analyze the query before it is searched. + +Full text functions can be used to match <>. +A multivalued field that contains a value that matches a full text query is considered to match the query. + +See <> for information on the limitations of full text search. + {esql} supports these full-text search functions: // tag::search_list[] diff --git a/docs/reference/esql/functions/search.asciidoc b/docs/reference/esql/functions/search.asciidoc index ae1b003b65abb..ba399ead8adfc 100644 --- a/docs/reference/esql/functions/search.asciidoc +++ b/docs/reference/esql/functions/search.asciidoc @@ -6,7 +6,10 @@ The only search operator is match (`:`). preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] -The match operator performs a <> on the specified field. Returns true if the provided query matches the row. +The match operator performs a <> on the specified field. +Returns true if the provided query matches the row. + +The match operator is equivalent to the <>. [.text-center] image::esql/functions/signature/match_operator.svg[Embedded,opts=inline] diff --git a/docs/reference/esql/functions/types/match.asciidoc b/docs/reference/esql/functions/types/match.asciidoc index 7523b29c62b1d..402277af44749 100644 --- a/docs/reference/esql/functions/types/match.asciidoc +++ b/docs/reference/esql/functions/types/match.asciidoc @@ -5,8 +5,33 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== field | query | result +boolean | boolean | boolean +boolean | keyword | boolean +date | date | boolean +date | keyword | boolean +date_nanos | date_nanos | boolean +date_nanos | keyword | boolean +double | double | boolean +double | integer | boolean +double | keyword | boolean +double | long | boolean +integer | double | boolean +integer | integer | boolean +integer | keyword | boolean +integer | long | boolean +ip | ip | boolean +ip | keyword | boolean keyword | keyword | boolean -keyword | text | boolean +long | double | boolean +long | integer | boolean +long | keyword | boolean +long | long | boolean text | keyword | boolean -text | text | boolean +unsigned_long | double | boolean +unsigned_long | integer | boolean +unsigned_long | keyword | boolean +unsigned_long | long | boolean +unsigned_long | unsigned_long | boolean +version | keyword | boolean +version | version | boolean |=== diff --git a/docs/reference/esql/functions/types/match_operator.asciidoc b/docs/reference/esql/functions/types/match_operator.asciidoc index 7523b29c62b1d..402277af44749 100644 --- a/docs/reference/esql/functions/types/match_operator.asciidoc +++ b/docs/reference/esql/functions/types/match_operator.asciidoc @@ -5,8 +5,33 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== field | query | result +boolean | boolean | boolean +boolean | keyword | boolean +date | date | boolean +date | keyword | boolean +date_nanos | date_nanos | boolean +date_nanos | keyword | boolean +double | double | boolean +double | integer | boolean +double | keyword | boolean +double | long | boolean +integer | double | boolean +integer | integer | boolean +integer | keyword | boolean +integer | long | boolean +ip | ip | boolean +ip | keyword | boolean keyword | keyword | boolean -keyword | text | boolean +long | double | boolean +long | integer | boolean +long | keyword | boolean +long | long | boolean text | keyword | boolean -text | text | boolean +unsigned_long | double | boolean +unsigned_long | integer | boolean +unsigned_long | keyword | boolean +unsigned_long | long | boolean +unsigned_long | unsigned_long | boolean +version | keyword | boolean +version | version | boolean |=== diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 26040529b04df..fb37fb3575551 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -5,8 +5,6 @@ * 2.0. */ -import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask import org.elasticsearch.gradle.util.GradleUtils @@ -95,5 +93,6 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("esql/80_text/values function", "The output type changed from TEXT to KEYWORD.") task.skipTest("privileges/11_builtin/Test get builtin privileges" ,"unnecessary to test compatibility") task.skipTest("esql/61_enrich_ip/Invalid IP strings", "We switched from exceptions to null+warnings for ENRICH runtime errors") + task.skipTest("esql/180_match_operator/match with non text field", "Match operator can now be used on non-text fields") }) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index f9d8cf00695c1..34af1edb9f99b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -52,6 +52,11 @@ public class CsvTestsDataLoader { private static final int BULK_DATA_SIZE = 100_000; private static final TestsDataset EMPLOYEES = new TestsDataset("employees", "mapping-default.json", "employees.csv").noSubfields(); + private static final TestsDataset EMPLOYEES_INCOMPATIBLE = new TestsDataset( + "employees_incompatible", + "mapping-default-incompatible.json", + "employees_incompatible.csv" + ).noSubfields(); private static final TestsDataset HOSTS = new TestsDataset("hosts"); private static final TestsDataset APPS = new TestsDataset("apps"); private static final TestsDataset APPS_SHORT = APPS.withIndex("apps_short").withTypeMapping(Map.of("id", "short")); @@ -103,6 +108,7 @@ public class CsvTestsDataLoader { public static final Map CSV_DATASET_MAP = Map.ofEntries( Map.entry(EMPLOYEES.indexName, EMPLOYEES), + Map.entry(EMPLOYEES_INCOMPATIBLE.indexName, EMPLOYEES_INCOMPATIBLE), Map.entry(HOSTS.indexName, HOSTS), Map.entry(APPS.indexName, APPS), Map.entry(APPS_SHORT.indexName, APPS_SHORT), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 5535e801b1b0c..18ce9d7e3e057 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -121,6 +121,7 @@ import static org.elasticsearch.test.ESTestCase.randomLong; import static org.elasticsearch.test.ESTestCase.randomLongBetween; import static org.elasticsearch.test.ESTestCase.randomMillisUpToYear9999; +import static org.elasticsearch.test.ESTestCase.randomNonNegativeLong; import static org.elasticsearch.test.ESTestCase.randomShort; import static org.elasticsearch.test.ESTestCase.randomZone; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; @@ -728,7 +729,8 @@ public static Literal randomLiteral(DataType type) { case BYTE -> randomByte(); case SHORT -> randomShort(); case INTEGER, COUNTER_INTEGER -> randomInt(); - case UNSIGNED_LONG, LONG, COUNTER_LONG -> randomLong(); + case LONG, COUNTER_LONG -> randomLong(); + case UNSIGNED_LONG -> randomNonNegativeLong(); case DATE_PERIOD -> Period.of(randomIntBetween(-1000, 1000), randomIntBetween(-13, 13), randomIntBetween(-32, 32)); case DATETIME -> randomMillisUpToYear9999(); case DATE_NANOS -> randomLongBetween(0, Long.MAX_VALUE); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/SpecReader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/SpecReader.java index 793268f18184d..760768cd0f118 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/SpecReader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/SpecReader.java @@ -86,7 +86,7 @@ public static List readURLSpec(URL source, Parser parser) throws Excep lineNumber++; } if (testName != null) { - throw new IllegalStateException("Read a test without a body at the end of [" + fileName + "]."); + throw new IllegalStateException("Read a test [" + testName + "] without a body at the end of [" + fileName + "]."); } } assertNull("Cannot find spec for test " + testName, testName); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees_incompatible.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees_incompatible.csv new file mode 100644 index 0000000000000..ddbdb89476c4c --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees_incompatible.csv @@ -0,0 +1,101 @@ +birth_date:date_nanos ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword +1953-09-02T00:00:00Z,10001,Georgi ,M,1986-06-26T00:00:00Z,2,2,2,2,Facello ,57305,2.03,2.03,2.03,2.03,true ,268728049,[Senior Python Developer,Accountant],[false,true],[1.19],[1],[1],[1.19] +1964-06-02T00:00:00Z,10002,Bezalel ,F,1985-11-21T00:00:00Z,5,5,5,5,Simmel ,56371,2.08,2.08,2.08,2.08,true ,328922887,[Senior Team Lead],[false,false],[-7.23,11.17],[-7,11],[-7,11],[-7.23,11.17] +1959-12-03T00:00:00Z,10003,Parto ,M,1986-08-28T00:00:00Z,4,4,4,4,Bamford ,61805,1.83,1.83,1.83,1.83,false,200296405,[],[],[14.68,12.82],[14,12],[14,12],[14.68,12.82] +1954-05-01T00:00:00Z,10004,Chirstian ,M,1986-12-01T00:00:00Z,5,5,5,5,Koblick ,36174,1.78,1.78,1.78,1.78,true ,311267831,[Reporting Analyst,Tech Lead,Head Human Resources,Support Engineer],[true],[3.65,-0.35,1.13,13.48],[3,0,1,13],[3,0,1,13],[3.65,-0.35,1.13,13.48] +1955-01-21T00:00:00Z,10005,Kyoichi ,M,1989-09-12T00:00:00Z,1,1,1,1,Maliniak ,63528,2.05,2.05,2.05,2.05,true ,244294991,[],[false,false,false,true],[-2.14,13.07],[-2,13],[-2,13],[-2.14,13.07] +1953-04-20T00:00:00Z,10006,Anneke ,F,1989-06-02T00:00:00Z,3,3,3,3,Preusig ,60335,1.56,1.56,1.56,1.56,false,372957040,[Tech Lead,Principal Support Engineer,Senior Team Lead],[],[-3.90],[-3],[-3],[-3.90] +1957-05-23T00:00:00Z,10007,Tzvetan ,F,1989-02-10T00:00:00Z,4,4,4,4,Zielinski ,74572,1.70,1.70,1.70,1.70,true ,393084805,[],[true,false,true,false],[-7.06,1.99,0.57],[-7,1,0],[-7,1,0],[-7.06,1.99,0.57] +1958-02-19T00:00:00Z,10008,Saniya ,M,1994-09-15T00:00:00Z,2,2,2,2,Kalloufi ,43906,2.10,2.10,2.10,2.10,true ,283074758,[Senior Python Developer,Junior Developer,Purchase Manager,Internship],[true,false],[12.68,3.54,0.75,-2.92],[12,3,0,-2],[12,3,0,-2],[12.68,3.54,0.75,-2.92] +1952-04-19T00:00:00Z,10009,Sumant ,F,1985-02-18T00:00:00Z,1,1,1,1,Peac ,66174,1.85,1.85,1.85,1.85,false,236805489,[Senior Python Developer,Internship],[],[],[],[],[] +1963-06-01T00:00:00Z,10010,Duangkaew , ,1989-08-24T00:00:00Z,4,4,4,4,Piveteau ,45797,1.70,1.70,1.70,1.70,false,315236372,[Architect,Reporting Analyst,Tech Lead,Purchase Manager],[true,true,false,false],[5.05,-6.77,4.69,12.15],[5,-6,4,12],[5,-6,4,12],[5.05,-6.77,4.69,12.15] +1953-11-07T00:00:00Z,10011,Mary , ,1990-01-22T00:00:00Z,5,5,5,5,Sluis ,31120,1.50,1.50,1.50,1.50,true ,239615525,[Architect,Reporting Analyst,Tech Lead,Senior Team Lead],[true,true],[10.35,-7.82,8.73,3.48],[10,-7,8,3],[10,-7,8,3],[10.35,-7.82,8.73,3.48] +1960-10-04T00:00:00Z,10012,Patricio , ,1992-12-18T00:00:00Z,5,5,5,5,Bridgland ,48942,1.97,1.97,1.97,1.97,false,365510850,[Head Human Resources,Accountant],[false,true,true,false],[0.04],[0],[0],[0.04] +1963-06-07T00:00:00Z,10013,Eberhardt , ,1985-10-20T00:00:00Z,1,1,1,1,Terkki ,48735,1.94,1.94,1.94,1.94,true ,253864340,[Reporting Analyst],[true,true],[],[],[],[] +1956-02-12T00:00:00Z,10014,Berni , ,1987-03-11T00:00:00Z,5,5,5,5,Genin ,37137,1.99,1.99,1.99,1.99,false,225049139,[Reporting Analyst,Data Scientist,Head Human Resources],[],[-1.89,9.07],[-1,9],[-1,9],[-1.89,9.07] +1959-08-19T00:00:00Z,10015,Guoxiang , ,1987-07-02T00:00:00Z,5,5,5,5,Nooteboom ,25324,1.66,1.66,1.66,1.66,true ,390266432,[Principal Support Engineer,Junior Developer,Head Human Resources,Support Engineer],[true,false,false,false],[14.25,12.40],[14,12],[14,12],[14.25,12.40] +1961-05-02T00:00:00Z,10016,Kazuhito , ,1995-01-27T00:00:00Z,2,2,2,2,Cappelletti ,61358,1.54,1.54,1.54,1.54,false,253029411,[Reporting Analyst,Python Developer,Accountant,Purchase Manager],[false,false],[-5.18,7.69],[-5,7],[-5,7],[-5.18,7.69] +1958-07-06T00:00:00Z,10017,Cristinel , ,1993-08-03T00:00:00Z,2,2,2,2,Bouloucos ,58715,1.74,1.74,1.74,1.74,false,236703986,[Data Scientist,Head Human Resources,Purchase Manager],[true,false,true,true],[-6.33],[-6],[-6],[-6.33] +1954-06-19T00:00:00Z,10018,Kazuhide , ,1987-04-03T00:00:00Z,2,2,2,2,Peha ,56760,1.97,1.97,1.97,1.97,false,309604079,[Junior Developer],[false,false,true,true],[-1.64,11.51,-5.32],[-1,11,-5],[-1,11,-5],[-1.64,11.51,-5.32] +1953-01-23T00:00:00Z,10019,Lillian , ,1999-04-30T00:00:00Z,1,1,1,1,Haddadi ,73717,2.06,2.06,2.06,2.06,false,342855721,[Purchase Manager],[false,false],[-6.84,8.42,-7.26],[-6,8,-7],[-6,8,-7],[-6.84,8.42,-7.26] +1952-12-24T00:00:00Z,10020,Mayuko ,M,1991-01-26T00:00:00Z, , , , ,Warwick ,40031,1.41,1.41,1.41,1.41,false,373309605,[Tech Lead],[true,true,false],[-5.81],[-5],[-5],[-5.81] +1960-02-20T00:00:00Z,10021,Ramzi ,M,1988-02-10T00:00:00Z, , , , ,Erde ,60408,1.47,1.47,1.47,1.47,false,287654610,[Support Engineer],[true],[],[],[],[] +1952-07-08T00:00:00Z,10022,Shahaf ,M,1995-08-22T00:00:00Z, , , , ,Famili ,48233,1.82,1.82,1.82,1.82,false,233521306,[Reporting Analyst,Data Scientist,Python Developer,Internship],[true,false],[12.09,2.85],[12,2],[12,2],[12.09,2.85] +1953-09-29T00:00:00Z,10023,Bojan ,F,1989-12-17T00:00:00Z, , , , ,Montemayor ,47896,1.75,1.75,1.75,1.75,true ,330870342,[Accountant,Support Engineer,Purchase Manager],[true,true,false],[14.63,0.80],[14,0],[14,0],[14.63,0.80] +1958-09-05T00:00:00Z,10024,Suzette ,F,1997-05-19T00:00:00Z, , , , ,Pettey ,64675,2.08,2.08,2.08,2.08,true ,367717671,[Junior Developer],[true,true,true,true],[],[],[],[] +1958-10-31T00:00:00Z,10025,Prasadram ,M,1987-08-17T00:00:00Z, , , , ,Heyers ,47411,1.87,1.87,1.87,1.87,false,371270797,[Accountant],[true,false],[-4.33,-2.90,12.06,-3.46],[-4,-2,12,-3],[-4,-2,12,-3],[-4.33,-2.90,12.06,-3.46] +1953-04-03T00:00:00Z,10026,Yongqiao ,M,1995-03-20T00:00:00Z, , , , ,Berztiss ,28336,2.10,2.10,2.10,2.10,true ,359208133,[Reporting Analyst],[false,true],[-7.37,10.62,11.20],[-7,10,11],[-7,10,11],[-7.37,10.62,11.20] +1962-07-10T00:00:00Z,10027,Divier ,F,1989-07-07T00:00:00Z, , , , ,Reistad ,73851,1.53,1.53,1.53,1.53,false,374037782,[Senior Python Developer],[false],[],[],[],[] +1963-11-26T00:00:00Z,10028,Domenick ,M,1991-10-22T00:00:00Z, , , , ,Tempesti ,39356,2.07,2.07,2.07,2.07,true ,226435054,[Tech Lead,Python Developer,Accountant,Internship],[true,false,false,true],[],[],[],[] +1956-12-13T00:00:00Z,10029,Otmar ,M,1985-11-20T00:00:00Z, , , , ,Herbst ,74999,1.99,1.99,1.99,1.99,false,257694181,[Senior Python Developer,Data Scientist,Principal Support Engineer],[true],[-0.32,-1.90,-8.19],[0,-1,-8],[0,-1,-8],[-0.32,-1.90,-8.19] +1958-07-14T00:00:00Z,10030, ,M,1994-02-17T00:00:00Z,3,3,3,3,Demeyer ,67492,1.92,1.92,1.92,1.92,false,394597613,[Tech Lead,Data Scientist,Senior Team Lead],[true,false,false],[-0.40],[0],[0],[-0.40] +1959-01-27T00:00:00Z,10031, ,M,1991-09-01T00:00:00Z,4,4,4,4,Joslin ,37716,1.68,1.68,1.68,1.68,false,348545109,[Architect,Senior Python Developer,Purchase Manager,Senior Team Lead],[false],[],[],[],[] +1960-08-09T00:00:00Z,10032, ,F,1990-06-20T00:00:00Z,3,3,3,3,Reistad ,62233,2.10,2.10,2.10,2.10,false,277622619,[Architect,Senior Python Developer,Junior Developer,Purchase Manager],[false,false],[9.32,-4.92],[9,-4],[9,-4],[9.32,-4.92] +1956-11-14T00:00:00Z,10033, ,M,1987-03-18T00:00:00Z,1,1,1,1,Merlo ,70011,1.63,1.63,1.63,1.63,false,208374744,[],[true],[],[],[],[] +1962-12-29T00:00:00Z,10034, ,M,1988-09-21T00:00:00Z,1,1,1,1,Swan ,39878,1.46,1.46,1.46,1.46,false,214393176,[Business Analyst,Data Scientist,Python Developer,Accountant],[false],[-8.46],[-8],[-8],[-8.46] +1953-02-08T00:00:00Z,10035, ,M,1988-09-05T00:00:00Z,5,5,5,5,Chappelet ,25945,1.81,1.81,1.81,1.81,false,203838153,[Senior Python Developer,Data Scientist],[false],[-2.54,-6.58],[-2,-6],[-2,-6],[-2.54,-6.58] +1959-08-10T00:00:00Z,10036, ,M,1992-01-03T00:00:00Z,4,4,4,4,Portugali ,60781,1.61,1.61,1.61,1.61,false,305493131,[Senior Python Developer],[true,false,false],[],[],[],[] +1963-07-22T00:00:00Z,10037, ,M,1990-12-05T00:00:00Z,2,2,2,2,Makrucki ,37691,2.00,2.00,2.00,2.00,true ,359217000,[Senior Python Developer,Tech Lead,Accountant],[false],[-7.08],[-7],[-7],[-7.08] +1960-07-20T00:00:00Z,10038, ,M,1989-09-20T00:00:00Z,4,4,4,4,Lortz ,35222,1.53,1.53,1.53,1.53,true ,314036411,[Senior Python Developer,Python Developer,Support Engineer],[],[],[],[],[] +1959-10-01T00:00:00Z,10039, ,M,1988-01-19T00:00:00Z,2,2,2,2,Brender ,36051,1.55,1.55,1.55,1.55,false,243221262,[Business Analyst,Python Developer,Principal Support Engineer],[true,true],[-6.90],[-6],[-6],[-6.90] + ,10040,Weiyi ,F,1993-02-14T00:00:00Z,4,4,4,4,Meriste ,37112,1.90,1.90,1.90,1.90,false,244478622,[Principal Support Engineer],[true,false,true,true],[6.97,14.74,-8.94,1.92],[6,14,-8,1],[6,14,-8,1],[6.97,14.74,-8.94,1.92] + ,10041,Uri ,F,1989-11-12T00:00:00Z,1,1,1,1,Lenart ,56415,1.75,1.75,1.75,1.75,false,287789442,[Data Scientist,Head Human Resources,Internship,Senior Team Lead],[],[9.21,0.05,7.29,-2.94],[9,0,7,-2],[9,0,7,-2],[9.21,0.05,7.29,-2.94] + ,10042,Magy ,F,1993-03-21T00:00:00Z,3,3,3,3,Stamatiou ,30404,1.44,1.44,1.44,1.44,true ,246355863,[Architect,Business Analyst,Junior Developer,Internship],[],[-9.28,9.42],[-9,9],[-9,9],[-9.28,9.42] + ,10043,Yishay ,M,1990-10-20T00:00:00Z,1,1,1,1,Tzvieli ,34341,1.52,1.52,1.52,1.52,true ,287222180,[Data Scientist,Python Developer,Support Engineer],[false,true,true],[-5.17,4.62,7.42],[-5,4,7],[-5,4,7],[-5.17,4.62,7.42] + ,10044,Mingsen ,F,1994-05-21T00:00:00Z,1,1,1,1,Casley ,39728,2.06,2.06,2.06,2.06,false,387408356,[Tech Lead,Principal Support Engineer,Accountant,Support Engineer],[true,true],[8.09],[8],[8],[8.09] + ,10045,Moss ,M,1989-09-02T00:00:00Z,3,3,3,3,Shanbhogue ,74970,1.70,1.70,1.70,1.70,false,371418933,[Principal Support Engineer,Junior Developer,Accountant,Purchase Manager],[true,false],[],[],[],[] + ,10046,Lucien ,M,1992-06-20T00:00:00Z,4,4,4,4,Rosenbaum ,50064,1.52,1.52,1.52,1.52,true ,302353405,[Principal Support Engineer,Junior Developer,Head Human Resources,Internship],[true,true,false,true],[2.39],[2],[2],[2.39] + ,10047,Zvonko ,M,1989-03-31T00:00:00Z,4,4,4,4,Nyanchama ,42716,1.52,1.52,1.52,1.52,true ,306369346,[Architect,Data Scientist,Principal Support Engineer,Senior Team Lead],[true],[-6.36,12.12],[-6,12],[-6,12],[-6.36,12.12] + ,10048,Florian ,M,1985-02-24T00:00:00Z,3,3,3,3,Syrotiuk ,26436,2.00,2.00,2.00,2.00,false,248451647,[Internship],[true,true],[],[],[],[] + ,10049,Basil ,F,1992-05-04T00:00:00Z,5,5,5,5,Tramer ,37853,1.52,1.52,1.52,1.52,true ,320725709,[Senior Python Developer,Business Analyst],[],[-1.05],[-1],[-1],[-1.05] +1958-05-21T00:00:00Z,10050,Yinghua ,M,1990-12-25T00:00:00Z,2,2,2,2,Dredge ,43026,1.96,1.96,1.96,1.96,true ,242731798,[Reporting Analyst,Junior Developer,Accountant,Support Engineer],[true],[8.70,10.94],[8,10],[8,10],[8.70,10.94] +1953-07-28T00:00:00Z,10051,Hidefumi ,M,1992-10-15T00:00:00Z,3,3,3,3,Caine ,58121,1.89,1.89,1.89,1.89,true ,374753122,[Business Analyst,Accountant,Purchase Manager],[],[],[],[],[] +1961-02-26T00:00:00Z,10052,Heping ,M,1988-05-21T00:00:00Z,1,1,1,1,Nitsch ,55360,1.79,1.79,1.79,1.79,true ,299654717,[],[true,true,false],[-0.55,-1.89,-4.22,-6.03],[0,-1,-4,-6],[0,-1,-4,-6],[-0.55,-1.89,-4.22,-6.03] +1954-09-13T00:00:00Z,10053,Sanjiv ,F,1986-02-04T00:00:00Z,3,3,3,3,Zschoche ,54462,1.58,1.58,1.58,1.58,false,368103911,[Support Engineer],[true,false,true,false],[-7.67,-3.25],[-7,-3],[-7,-3],[-7.67,-3.25] +1957-04-04T00:00:00Z,10054,Mayumi ,M,1995-03-13T00:00:00Z,4,4,4,4,Schueller ,65367,1.82,1.82,1.82,1.82,false,297441693,[Principal Support Engineer],[false,false],[],[],[],[] +1956-06-06T00:00:00Z,10055,Georgy ,M,1992-04-27T00:00:00Z,5,5,5,5,Dredge ,49281,2.04,2.04,2.04,2.04,false,283157844,[Senior Python Developer,Head Human Resources,Internship,Support Engineer],[false,false,true],[7.34,12.99,3.17],[7,12,3],[7,12,3],[7.34,12.99,3.17] +1961-09-01T00:00:00Z,10056,Brendon ,F,1990-02-01T00:00:00Z,2,2,2,2,Bernini ,33370,1.57,1.57,1.57,1.57,true ,349086555,[Senior Team Lead],[true,false,false],[10.99,-5.17],[10,-5],[10,-5],[10.99,-5.17] +1954-05-30T00:00:00Z,10057,Ebbe ,F,1992-01-15T00:00:00Z,4,4,4,4,Callaway ,27215,1.59,1.59,1.59,1.59,true ,324356269,[Python Developer,Head Human Resources],[],[-6.73,-2.43,-5.27,1.03],[-6,-2,-5,1],[-6,-2,-5,1],[-6.73,-2.43,-5.27,1.03] +1954-10-01T00:00:00Z,10058,Berhard ,M,1987-04-13T00:00:00Z,3,3,3,3,McFarlin ,38376,1.83,1.83,1.83,1.83,false,268378108,[Principal Support Engineer],[],[-4.89],[-4],[-4],[-4.89] +1953-09-19T00:00:00Z,10059,Alejandro ,F,1991-06-26T00:00:00Z,2,2,2,2,McAlpine ,44307,1.48,1.48,1.48,1.48,false,237368465,[Architect,Principal Support Engineer,Purchase Manager,Senior Team Lead],[false],[5.53,13.38,-4.69,6.27],[5,13,-4,6],[5,13,-4,6],[5.53,13.38,-4.69,6.27] +1961-10-15T00:00:00Z,10060,Breannda ,M,1987-11-02T00:00:00Z,2,2,2,2,Billingsley ,29175,1.42,1.42,1.42,1.42,true ,341158890,[Business Analyst,Data Scientist,Senior Team Lead],[false,false,true,false],[-1.76,-0.85],[-1,0],[-1,0],[-1.76,-0.85] +1962-10-19T00:00:00Z,10061,Tse ,M,1985-09-17T00:00:00Z,1,1,1,1,Herber ,49095,1.45,1.45,1.45,1.45,false,327550310,[Purchase Manager,Senior Team Lead],[false,true],[14.39,-2.58,-0.95],[14,-2,0],[14,-2,0],[14.39,-2.58,-0.95] +1961-11-02T00:00:00Z,10062,Anoosh ,M,1991-08-30T00:00:00Z,3,3,3,3,Peyn ,65030,1.70,1.70,1.70,1.70,false,203989706,[Python Developer,Senior Team Lead],[false,true,true],[-1.17],[-1],[-1],[-1.171] +1952-08-06T00:00:00Z,10063,Gino ,F,1989-04-08T00:00:00Z,3,3,3,3,Leonhardt ,52121,1.78,1.78,1.78,1.78,true ,214068302,[],[true],[],[],[],[] +1959-04-07T00:00:00Z,10064,Udi ,M,1985-11-20T00:00:00Z,5,5,5,5,Jansch ,33956,1.93,1.93,1.93,1.93,false,307364077,[Purchase Manager],[false,false,true,false],[-8.66,-2.52],[-8,-2],[-8,-2],[-8.66,-2.52] +1963-04-14T00:00:00Z,10065,Satosi ,M,1988-05-18T00:00:00Z,2,2,2,2,Awdeh ,50249,1.59,1.59,1.59,1.59,false,372660279,[Business Analyst,Data Scientist,Principal Support Engineer],[false,true],[-1.47,14.44,-9.81],[-1,14,-9],[-1,14,-9],[-1.47,14.44,-9.81] +1952-11-13T00:00:00Z,10066,Kwee ,M,1986-02-26T00:00:00Z,5,5,5,5,Schusler ,31897,2.10,2.10,2.10,2.10,true ,360906451,[Senior Python Developer,Data Scientist,Accountant,Internship],[true,true,true],[5.94],[5],[5],[5.94] +1953-01-07T00:00:00Z,10067,Claudi ,M,1987-03-04T00:00:00Z,2,2,2,2,Stavenow ,52044,1.77,1.77,1.77,1.77,true ,347664141,[Tech Lead,Principal Support Engineer],[false,false],[8.72,4.44],[8,4],[8,4],[8.72,4.44] +1962-11-26T00:00:00Z,10068,Charlene ,M,1987-08-07T00:00:00Z,3,3,3,3,Brattka ,28941,1.58,1.58,1.58,1.58,true ,233999584,[Architect],[true],[3.43,-5.61,-5.29],[3,-5,-5],[3,-5,-5],[3.43,-5.61,-5.29] +1960-09-06T00:00:00Z,10069,Margareta ,F,1989-11-05T00:00:00Z,5,5,5,5,Bierman ,41933,1.77,1.77,1.77,1.77,true ,366512352,[Business Analyst,Junior Developer,Purchase Manager,Support Engineer],[false],[-3.34,-6.33,6.23,-0.31],[-3,-6,6,0],[-3,-6,6,0],[-3.34,-6.33,6.23,-0.31] +1955-08-20T00:00:00Z,10070,Reuven ,M,1985-10-14T00:00:00Z,3,3,3,3,Garigliano ,54329,1.77,1.77,1.77,1.77,true ,347188604,[],[true,true,true],[-5.90],[-5],[-5],[-5.90] +1958-01-21T00:00:00Z,10071,Hisao ,M,1987-10-01T00:00:00Z,2,2,2,2,Lipner ,40612,2.07,2.07,2.07,2.07,false,306671693,[Business Analyst,Reporting Analyst,Senior Team Lead],[false,false,false],[-2.69],[-2],[-2],[-2.69] +1952-05-15T00:00:00Z,10072,Hironoby ,F,1988-07-21T00:00:00Z,5,5,5,5,Sidou ,54518,1.82,1.82,1.82,1.82,true ,209506065,[Architect,Tech Lead,Python Developer,Senior Team Lead],[false,false,true,false],[11.21,-2.30,2.22,-5.44],[11,-2,2,-5],[11,-2,2,-5],[11.21,-2.30,2.22,-5.44] +1954-02-23T00:00:00Z,10073,Shir ,M,1991-12-01T00:00:00Z,4,4,4,4,McClurg ,32568,1.66,1.66,1.66,1.66,false,314930367,[Principal Support Engineer,Python Developer,Junior Developer,Purchase Manager],[true,false],[-5.67],[-5],[-5],[-5.67] +1955-08-28T00:00:00Z,10074,Mokhtar ,F,1990-08-13T00:00:00Z,5,5,5,5,Bernatsky ,38992,1.64,1.64,1.64,1.64,true ,382397583,[Senior Python Developer,Python Developer],[true,false,false,true],[6.70,1.98,-5.64,2.96],[6,1,-5,2],[6,1,-5,2],[6.70,1.98,-5.64,2.96] +1960-03-09T00:00:00Z,10075,Gao ,F,1987-03-19T00:00:00Z,5,5,5,5,Dolinsky ,51956,1.94,1.94,1.94,1.94,false,370238919,[Purchase Manager],[true],[9.63,-3.29,8.42],[9,-3,8],[9,-3,8],[9.63,-3.29,8.42] +1952-06-13T00:00:00Z,10076,Erez ,F,1985-07-09T00:00:00Z,3,3,3,3,Ritzmann ,62405,1.83,1.83,1.83,1.83,false,376240317,[Architect,Senior Python Developer],[false],[-6.90,-1.30,8.75],[-6,-1,8],[-6,-1,8],[-6.90,-1.30,8.75] +1964-04-18T00:00:00Z,10077,Mona ,M,1990-03-02T00:00:00Z,5,5,5,5,Azuma ,46595,1.68,1.68,1.68,1.68,false,351960222,[Internship],[],[-0.01],[0],[0],[-0.01] +1959-12-25T00:00:00Z,10078,Danel ,F,1987-05-26T00:00:00Z,2,2,2,2,Mondadori ,69904,1.81,1.81,1.81,1.81,true ,377116038,[Architect,Principal Support Engineer,Internship],[true],[-7.88,9.98,12.52],[-7,9,12],[-7,9,12],[-7.88,9.98,12.52] +1961-10-05T00:00:00Z,10079,Kshitij ,F,1986-03-27T00:00:00Z,2,2,2,2,Gils ,32263,1.59,1.59,1.59,1.59,false,320953330,[],[false],[7.58],[7],[7],[7.58] +1957-12-03T00:00:00Z,10080,Premal ,M,1985-11-19T00:00:00Z,5,5,5,5,Baek ,52833,1.80,1.80,1.80,1.80,false,239266137,[Senior Python Developer],[],[-4.35,7.36,5.56],[-4,7,5],[-4,7,5],[-4.35,7.36,5.56] +1960-12-17T00:00:00Z,10081,Zhongwei ,M,1986-10-30T00:00:00Z,2,2,2,2,Rosen ,50128,1.44,1.44,1.44,1.44,true ,321375511,[Accountant,Internship],[false,false,false],[],[],[],[] +1963-09-09T00:00:00Z,10082,Parviz ,M,1990-01-03T00:00:00Z,4,4,4,4,Lortz ,49818,1.61,1.61,1.61,1.61,false,232522994,[Principal Support Engineer],[false],[1.19,-3.39],[1,-3],[1,-3],[1.19,-3.39] +1959-07-23T00:00:00Z,10083,Vishv ,M,1987-03-31T00:00:00Z,1,1,1,1,Zockler ,39110,1.42,1.42,1.42,1.42,false,331236443,[Head Human Resources],[],[],[],[],[] +1960-05-25T00:00:00Z,10084,Tuval ,M,1995-12-15T00:00:00Z,1,1,1,1,Kalloufi ,28035,1.51,1.51,1.51,1.51,true ,359067056,[Principal Support Engineer],[false],[],[],[],[] +1962-11-07T00:00:00Z,10085,Kenroku ,M,1994-04-09T00:00:00Z,5,5,5,5,Malabarba ,35742,2.01,2.01,2.01,2.01,true ,353404008,[Senior Python Developer,Business Analyst,Tech Lead,Accountant],[],[11.67,6.75,8.40],[11,6,8],[11,6,8],[11.67,6.75,8.40] +1962-11-19T00:00:00Z,10086,Somnath ,M,1990-02-16T00:00:00Z,1,1,1,1,Foote ,68547,1.74,1.74,1.74,1.74,true ,328580163,[Senior Python Developer],[false,true],[13.61],[13],[13],[13.61] +1959-07-23T00:00:00Z,10087,Xinglin ,F,1986-09-08T00:00:00Z,5,5,5,5,Eugenio ,32272,1.74,1.74,1.74,1.74,true ,305782871,[Junior Developer,Internship],[false,false],[-2.05],[-2],[-2],[-2.05] +1954-02-25T00:00:00Z,10088,Jungsoon ,F,1988-09-02T00:00:00Z,5,5,5,5,Syrzycki ,39638,1.91,1.91,1.91,1.91,false,330714423,[Reporting Analyst,Business Analyst,Tech Lead],[true],[],[],[],[] +1963-03-21T00:00:00Z,10089,Sudharsan ,F,1986-08-12T00:00:00Z,4,4,4,4,Flasterstein,43602,1.57,1.57,1.57,1.57,true ,232951673,[Junior Developer,Accountant],[true,false,false,false],[],[],[],[] +1961-05-30T00:00:00Z,10090,Kendra ,M,1986-03-14T00:00:00Z,2,2,2,2,Hofting ,44956,2.03,2.03,2.03,2.03,true ,212460105,[],[false,false,false,true],[7.15,-1.85,3.60],[7,-1,3],[7,-1,3],[7.15,-1.85,3.60] +1955-10-04T00:00:00Z,10091,Amabile ,M,1992-11-18T00:00:00Z,3,3,3,3,Gomatam ,38645,2.09,2.09,2.09,2.09,true ,242582807,[Reporting Analyst,Python Developer],[true,true,false,false],[-9.23,7.50,5.85,5.19],[-9,7,5,5],[-9,7,5,5],[-9.23,7.50,5.85,5.19] +1964-10-18T00:00:00Z,10092,Valdiodio ,F,1989-09-22T00:00:00Z,1,1,1,1,Niizuma ,25976,1.75,1.75,1.75,1.75,false,313407352,[Junior Developer,Accountant],[false,false,true,true],[8.78,0.39,-6.77,8.30],[8,0,-6,8],[8,0,-6,8],[8.78,0.39,-6.77,8.30] +1964-06-11T00:00:00Z,10093,Sailaja ,M,1996-11-05T00:00:00Z,3,3,3,3,Desikan ,45656,1.69,1.69,1.69,1.69,false,315904921,[Reporting Analyst,Tech Lead,Principal Support Engineer,Purchase Manager],[],[-0.88],[0],[0],[-0.88] +1957-05-25T00:00:00Z,10094,Arumugam ,F,1987-04-18T00:00:00Z,5,5,5,5,Ossenbruggen,66817,2.10,2.10,2.10,2.10,false,332920135,[Senior Python Developer,Principal Support Engineer,Accountant],[true,false,true],[2.22,7.92],[2,7],[2,7],[2.22,7.92] +1965-01-03T00:00:00Z,10095,Hilari ,M,1986-07-15T00:00:00Z,4,4,4,4,Morton ,37702,1.55,1.55,1.55,1.55,false,321850475,[],[true,true,false,false],[-3.93,-6.66],[-3,-6],[-3,-6],[-3.93,-6.66] +1954-09-16T00:00:00Z,10096,Jayson ,M,1990-01-14T00:00:00Z,4,4,4,4,Mandell ,43889,1.94,1.94,1.94,1.94,false,204381503,[Architect,Reporting Analyst],[false,false,false],[],[],[],[] +1952-02-27T00:00:00Z,10097,Remzi ,M,1990-09-15T00:00:00Z,3,3,3,3,Waschkowski ,71165,1.53,1.53,1.53,1.53,false,206258084,[Reporting Analyst,Tech Lead],[true,false],[-1.12],[-1],[-1],[-1.12] +1961-09-23T00:00:00Z,10098,Sreekrishna,F,1985-05-13T00:00:00Z,4,4,4,4,Servieres ,44817,2.00,2.00,2.00,2.00,false,272392146,[Architect,Internship,Senior Team Lead],[false],[-2.83,8.31,4.38],[-2,8,4],[-2,8,4],[-2.83,8.31,4.38] +1956-05-25T00:00:00Z,10099,Valter ,F,1988-10-18T00:00:00Z,2,2,2,2,Sullins ,73578,1.81,1.81,1.81,1.81,true ,377713748,[],[true,true],[10.71,14.26,-8.78,-3.98],[10,14,-8,-3],[10,14,-8,-3],[10.71,14.26,-8.78,-3.98] +1953-04-21T00:00:00Z,10100,Hironobu ,F,1987-09-21T00:00:00Z,4,4,4,4,Haraldson ,68431,1.77,1.77,1.77,1.77,true ,223910853,[Purchase Manager],[false,true,true,false],[13.97,-7.49],[13,-7],[13,-7],[13.97,-7.49] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-all-types.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-all-types.json index ee1ef56a63dfb..04b59f347ebfc 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-all-types.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-all-types.json @@ -17,6 +17,9 @@ "date": { "type": "date" }, + "date_nanos": { + "type": "date_nanos" + }, "double": { "type": "double" }, diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic.json index 9ce87d01bfbb9..85e797aea86df 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic.json @@ -21,6 +21,9 @@ "_meta_field": { "type" : "keyword" }, + "hire_date": { + "type": "date" + }, "job": { "type": "text", "fields": { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-default-incompatible.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-default-incompatible.json new file mode 100644 index 0000000000000..607ae5c9ab2c8 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-default-incompatible.json @@ -0,0 +1,80 @@ +{ + "properties" : { + "emp_no" : { + "type" : "long" + }, + "first_name" : { + "type" : "text" + }, + "last_name" : { + "type" : "text" + }, + "gender" : { + "type" : "text" + }, + "birth_date": { + "type" : "date" + }, + "hire_date": { + "type" : "date_nanos" + }, + "salary" : { + "type" : "long" + }, + "languages" : { + "type" : "byte", + "fields": { + "long": { + "type": "long" + }, + "short": { + "type": "short" + }, + "int": { + "type": "integer" + } + } + }, + "height": { + "type" : "float", + "fields" : { + "double" : { + "type" : "double" + }, + "scaled_float": { + "type": "scaled_float", + "scaling_factor": 100 + }, + "half_float": { + "type": "half_float" + } + } + }, + "still_hired": { + "type" : "keyword" + }, + "avg_worked_seconds" : { + "type" : "unsigned_long" + }, + "job_positions" : { + "type" : "text" + }, + "is_rehired" : { + "type" : "keyword" + }, + "salary_change": { + "type": "float", + "fields": { + "int": { + "type": "integer" + }, + "long": { + "type": "long" + }, + "keyword": { + "type" : "keyword" + } + } + } + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec index c35f4c19cc347..03b24555dbeff 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec @@ -197,3 +197,329 @@ emp_no:integer | first_name:keyword | last_name:keyword 10041 | Uri | Lenart 10043 | Yishay | Tzvieli ; + +testMatchBooleanField +required_capability: match_function +required_capability: match_additional_types + +from employees +| where match(still_hired, true) and height > 2.08 +| keep first_name, still_hired, height; +ignoreOrder:true + +first_name:keyword | still_hired:boolean | height:double +Saniya | true | 2.1 +Yongqiao | true | 2.1 +Kwee | true | 2.1 +Amabile | true | 2.09 +; + +testMatchIntegerField +required_capability: match_function +required_capability: match_additional_types + +from employees +| where match(emp_no, 10004) +| keep emp_no, first_name; + +emp_no:integer | first_name:keyword +10004 | Chirstian +; + +testMatchDoubleField +required_capability: match_function +required_capability: match_additional_types + +from employees +| where match(salary_change, 9.07) +| keep emp_no, salary_change; + +emp_no:integer | salary_change:double +10014 | [-1.89, 9.07] +; + +testMatchLongField +required_capability: match_function +required_capability: match_additional_types + +from date_nanos +| where match(num, 1698069301543123456) +| keep num; + +num:long +1698069301543123456 +; + +testMatchUnsignedLongField +required_capability: match_function +required_capability: match_additional_types + +from ul_logs +| where match(bytes_out, 12749081495402663265) +| keep bytes_out; + +bytes_out:unsigned_long +12749081495402663265 +; + +testMatchVersionField +required_capability: match_function +required_capability: match_additional_types + +from apps +| where match(version, "2.1"::VERSION) +| keep name, version; + +name:keyword | version:version +bbbbb | 2.1 +; + +testMatchIpField +required_capability: match_function +required_capability: match_additional_types + +from sample_data +| where match(client_ip, "172.21.0.5") +| keep client_ip, message; + +client_ip:ip | message:keyword +172.21.0.5 | Disconnected +; + +testMatchDateFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from date_nanos +| where match(millis, "2023-10-23T13:55:01.543Z") +| keep millis; + +millis:date +2023-10-23T13:55:01.543Z +; + +testMatchDateNanosFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from date_nanos +| where match(nanos, "2023-10-23T13:55:01.543123456Z") +| keep nanos; + +nanos:date_nanos +2023-10-23T13:55:01.543123456Z +; + +testMatchBooleanFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from employees +| where match(still_hired, "true") and height > 2.08 +| keep first_name, still_hired, height; +ignoreOrder:true + +first_name:keyword | still_hired:boolean | height:double +Saniya | true | 2.1 +Yongqiao | true | 2.1 +Kwee | true | 2.1 +Amabile | true | 2.09 +; + +testMatchIntegerFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from employees +| where match(emp_no, "10004") +| keep emp_no, first_name; + +emp_no:integer | first_name:keyword +10004 | Chirstian +; + +testMatchDoubleFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from employees +| where match(salary_change, "9.07") +| keep emp_no, salary_change; + +emp_no:integer | salary_change:double +10014 | [-1.89, 9.07] +; + +testMatchLongFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from date_nanos +| where match(num, "1698069301543123456") +| keep num; + +num:long +1698069301543123456 +; + +testMatchUnsignedLongFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from ul_logs +| where match(bytes_out, "12749081495402663265") +| keep bytes_out; + +bytes_out:unsigned_long +12749081495402663265 +; + +testMatchVersionFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from apps +| where match(version, "2.1") +| keep name, version; + +name:keyword | version:version +bbbbb | 2.1 +; + +testMatchIntegerAsDouble +required_capability: match_function +required_capability: match_additional_types + +from employees +| where match(emp_no, 10004.0) +| keep emp_no, first_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword +10004 | Chirstian +; + +testMatchDoubleAsIntegerField +required_capability: match_function +required_capability: match_additional_types + +from employees +| where match(height, 2) +| keep emp_no, height; +ignoreOrder:true + +emp_no:integer | height:double +10037 | 2.0 +10048 | 2.0 +10098 | 2.0 +; + +testMatchMultipleFieldTypesIntLong +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where match(emp_no::int, 10005) +| eval emp_as_int = emp_no::int +| eval name_as_kw = first_name::keyword +| keep emp_as_int, name_as_kw +; + +emp_as_int:integer | name_as_kw:keyword +10005 | Kyoichi +10005 | Kyoichi +; + +testMatchMultipleFieldTypesKeywordText +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where match(first_name::keyword, "Kazuhito") +| eval first_name_kwd = first_name::keyword +| keep first_name_kwd +; + +first_name_kwd:keyword +Kazuhito +Kazuhito +; + +testMatchMultipleFieldTypesDoubleFloat +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where match(height::double, 2.03) +| eval height_dbl = height::double +| eval emp_no = emp_no::int +| keep emp_no, height_dbl +; +ignoreOrder:true + +emp_no:integer | height_dbl:double +10001 | 2.0299999713897705 +10090 | 2.0299999713897705 +10001 | 2.03 +10090 | 2.03 +; + +testMatchMultipleFieldTypesBooleanKeyword +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where match(still_hired::keyword, "true") and height.scaled_float == 2.08 +| eval still_hired_bool = still_hired::boolean +| keep still_hired_bool +; + +still_hired_bool:boolean +true +true +true +true +; + +testMatchMultipleFieldTypesLongUnsignedLong +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where match(avg_worked_seconds::unsigned_long, 200296405) +| eval avg_worked_seconds_ul = avg_worked_seconds::unsigned_long +| keep avg_worked_seconds_ul +; + +avg_worked_seconds_ul:unsigned_long +200296405 +200296405 +; + +testMatchMultipleFieldTypesDateNanosDate +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where match(hire_date::datetime, "1986-06-26T00:00:00.000Z") +| eval hire_date_nanos = hire_date::date_nanos +| keep hire_date_nanos +; + +hire_date_nanos:date_nanos +1986-06-26T00:00:00.000Z +1986-06-26T00:00:00.000Z +; + +testMatchWithWrongFieldValue +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where match(still_hired::boolean, "Wrong boolean") +| eval emp_no_bool = emp_no::boolean +| keep emp_no_bool +; + +emp_no_bool:boolean +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec index 7b55ece964b89..56f7f5ccd8823 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec @@ -217,3 +217,319 @@ count(*): long | author.keyword:keyword 1 | Paul Faulkner 8 | William Faulkner ; + +testMatchBooleanField +required_capability: match_function +required_capability: match_additional_types + +from employees +| where still_hired:true and height > 2.08 +| keep first_name, still_hired, height; +ignoreOrder:true + +first_name:keyword | still_hired:boolean | height:double +Saniya | true | 2.1 +Yongqiao | true | 2.1 +Kwee | true | 2.1 +Amabile | true | 2.09 +; + +testMatchIntegerField +required_capability: match_function +required_capability: match_additional_types + +from employees +| where emp_no:10004 +| keep emp_no, first_name; + +emp_no:integer | first_name:keyword +10004 | Chirstian +; + +testMatchDoubleField +required_capability: match_function +required_capability: match_additional_types + +from employees +| where salary_change:9.07 +| keep emp_no, salary_change; + +emp_no:integer | salary_change:double +10014 | [-1.89, 9.07] +; + +testMatchLongField +required_capability: match_function +required_capability: match_additional_types + +from date_nanos +| where num:1698069301543123456 +| keep num; + +num:long +1698069301543123456 +; + +testMatchUnsignedLongField +required_capability: match_function +required_capability: match_additional_types + +from ul_logs +| where bytes_out:12749081495402663265 +| keep bytes_out; + +bytes_out:unsigned_long +12749081495402663265 +; + +testMatchIpFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from sample_data +| where client_ip:"172.21.0.5" +| keep client_ip, message; + +client_ip:ip | message:keyword +172.21.0.5 | Disconnected +; + +testMatchDateFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from date_nanos +| where millis:"2023-10-23T13:55:01.543Z" +| keep millis; + +millis:date +2023-10-23T13:55:01.543Z +; + +testMatchDateNanosFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from date_nanos +| where nanos:"2023-10-23T13:55:01.543123456Z" +| keep nanos; + +nanos:date_nanos +2023-10-23T13:55:01.543123456Z +; + +testMatchBooleanFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from employees +| where still_hired:"true" and height > 2.08 +| keep first_name, still_hired, height; +ignoreOrder:true + +first_name:keyword | still_hired:boolean | height:double +Saniya | true | 2.1 +Yongqiao | true | 2.1 +Kwee | true | 2.1 +Amabile | true | 2.09 +; + +testMatchIntegerFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from employees +| where emp_no:"10004" +| keep emp_no, first_name; + +emp_no:integer | first_name:keyword +10004 | Chirstian +; + +testMatchDoubleFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from employees +| where salary_change:"9.07" +| keep emp_no, salary_change; + +emp_no:integer | salary_change:double +10014 | [-1.89, 9.07] +; + +testMatchLongFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from date_nanos +| where num:"1698069301543123456" +| keep num; + +num:long +1698069301543123456 +; + +testMatchUnsignedLongFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from ul_logs +| where bytes_out:"12749081495402663265" +| keep bytes_out; + +bytes_out:unsigned_long +12749081495402663265 +; + +testMatchVersionFieldAsString +required_capability: match_function +required_capability: match_additional_types + +from apps +| where version:"2.1" +| keep name, version; + +name:keyword | version:version +bbbbb | 2.1 +; + +testMatchIntegerAsDouble +required_capability: match_function +required_capability: match_additional_types + +from employees +| where emp_no:10004.0 +| keep emp_no, first_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword +10004 | Chirstian +; + +testMatchDoubleAsIntegerField +required_capability: match_function +required_capability: match_additional_types + +from employees +| where height:2 +| keep emp_no, height; +ignoreOrder:true + +emp_no:integer | height:double +10037 | 2.0 +10048 | 2.0 +10098 | 2.0 +; + +testMatchMultipleFieldTypes +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where emp_no::int : 10005 +| eval emp_as_int = emp_no::int +| eval name_as_kw = first_name::keyword +| keep emp_as_int, name_as_kw +; + +emp_as_int:integer | name_as_kw:keyword +10005 | Kyoichi +10005 | Kyoichi +; + + +testMatchMultipleFieldTypesKeywordText +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where first_name::keyword : "Kazuhito" +| eval first_name_kwd = first_name::keyword +| keep first_name_kwd +; + +first_name_kwd:keyword +Kazuhito +Kazuhito +; + +testMatchMultipleFieldTypesDoubleFloat +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where height::double : 2.03 +| eval height_dbl = height::double +| eval emp_no = emp_no::int +| keep emp_no, height_dbl +; +ignoreOrder:true + +emp_no:integer | height_dbl:double +10001 | 2.0299999713897705 +10090 | 2.0299999713897705 +10001 | 2.03 +10090 | 2.03 +; + +testMatchMultipleFieldTypesBooleanKeyword +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where still_hired::keyword : "true" and height.scaled_float == 2.08 +| eval still_hired_bool = still_hired::boolean +| keep still_hired_bool +; + +still_hired_bool:boolean +true +true +true +true +; + +testMatchMultipleFieldTypesLongUnsignedLong +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where avg_worked_seconds::unsigned_long : 200296405 +| eval avg_worked_seconds_ul = avg_worked_seconds::unsigned_long +| keep avg_worked_seconds_ul +; + +avg_worked_seconds_ul:unsigned_long +200296405 +200296405 +; + +testMatchMultipleFieldTypesDateNanosDate +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where hire_date::datetime : "1986-06-26T00:00:00.000Z" +| eval hire_date_nanos = hire_date::date_nanos +| keep hire_date_nanos +; + +hire_date_nanos:date_nanos +1986-06-26T00:00:00.000Z +1986-06-26T00:00:00.000Z +; + +testMatchWithWrongFieldValue +required_capability: match_function +required_capability: match_additional_types + +from employees,employees_incompatible +| where still_hired::boolean : "Wrong boolean" +| eval emp_no_bool = emp_no::boolean +| keep emp_no_bool +; + +emp_no_bool:boolean +; + diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java index 99f7d48a0d636..58b1652653ca3 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java @@ -268,16 +268,6 @@ public void testMatchWithinEval() { assertThat(error.getMessage(), containsString("[:] operator is only supported in WHERE commands")); } - public void testMatchWithNonTextField() { - var query = """ - FROM test - | WHERE id:"fox" - """; - - var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("first argument of [id:\"fox\"] must be [string], found value [id] type [integer]")); - } - private void createAndPopulateIndex() { var indexName = "test"; var client = client().admin().indices(); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java index 6a360eb319abb..d0a641f086fe4 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; @@ -22,7 +21,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.CoreMatchers.containsString; -@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug") +//@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug") public class MatchOperatorIT extends AbstractEsqlIntegTestCase { @Before @@ -248,11 +247,15 @@ public void testMatchWithinEval() { public void testMatchWithNonTextField() { var query = """ FROM test - | WHERE id:"fox" + | WHERE id:3 + | KEEP id """; - var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("first argument of [id:\"fox\"] must be [string], found value [id] type [integer]")); + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValues(resp.values(), List.of(List.of(3))); + } } private void createAndPopulateIndex() { diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index f84cfe3060503..efc2e36609902 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -78,7 +78,7 @@ regexBooleanExpression ; matchBooleanExpression - : fieldExp=qualifiedName COLON queryString=constant + : fieldExp=qualifiedName (CAST_OP fieldType=dataType)? COLON matchQuery=constant ; valueExpression diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 7c3f2a45df6a0..1aee1df3dbafb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -560,7 +560,12 @@ public enum Cap { /** * Term function */ - TERM_FUNCTION(Build.current().isSnapshot()); + TERM_FUNCTION(Build.current().isSnapshot()), + + /** + * Additional types for match function and operator + */ + MATCH_ADDITIONAL_TYPES; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index 9addf08e1b5f9..78dc05af8f342 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -17,7 +17,6 @@ import java.util.List; -import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNullAndFoldable; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -47,7 +46,16 @@ protected final TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - return resolveNonQueryParamTypes().and(resolveQueryParamType()); + return resolveNonQueryParamTypes().and(resolveQueryParamType().and(checkParamCompatibility())); + } + + /** + * Checks parameter specific compatibility, to be overriden by subclasses + * + * @return TypeResolution for param compatibility + */ + protected TypeResolution checkParamCompatibility() { + return TypeResolution.TYPE_RESOLVED; } /** @@ -55,7 +63,7 @@ protected final TypeResolution resolveType() { * * @return type resolution for query parameter */ - private TypeResolution resolveQueryParamType() { + protected TypeResolution resolveQueryParamType() { return isString(query(), sourceText(), queryParamOrdinal()).and(isNotNullAndFoldable(query(), sourceText(), queryParamOrdinal())); } @@ -73,19 +81,17 @@ public Expression query() { } /** - * Returns the resulting query as a String + * Returns the resulting query as an object * - * @return query expression as a string + * @return query expression as an object */ - public final String queryAsText() { + public Object queryAsObject() { Object queryAsObject = query().fold(); if (queryAsObject instanceof BytesRef bytesRef) { return bytesRef.utf8ToString(); } - throw new IllegalArgumentException( - format(null, "{} argument in {} function needs to be resolved to a string", queryParamOrdinal(), functionName()) - ); + return queryAsObject; } /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java index 522a5574c0053..2b9a7c73a5853 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -19,19 +20,37 @@ import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.NumericUtils; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; import java.io.IOException; import java.util.List; import java.util.Locale; +import java.util.Set; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNull; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNullAndFoldable; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; +import static org.elasticsearch.xpack.esql.core.type.DataType.IP; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; +import static org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison.formatIncompatibleTypesMessage; /** * Full text function that performs a {@link QueryStringQuery} . @@ -44,19 +63,50 @@ public class Match extends FullTextFunction implements Validatable { private transient Boolean isOperator; + public static final Set FIELD_DATA_TYPES = Set.of( + KEYWORD, + TEXT, + BOOLEAN, + DATETIME, + DATE_NANOS, + DOUBLE, + INTEGER, + IP, + LONG, + UNSIGNED_LONG, + VERSION + ); + public static final Set QUERY_DATA_TYPES = Set.of( + KEYWORD, + BOOLEAN, + DATETIME, + DATE_NANOS, + DOUBLE, + INTEGER, + IP, + LONG, + UNSIGNED_LONG, + VERSION + ); + @FunctionInfo( returnType = "boolean", preview = true, - description = "Performs a match query on the specified field. Returns true if the provided query matches the row.", + description = "Performs a <> on the specified field. " + + "Returns true if the provided query matches the row.", examples = { @Example(file = "match-function", tag = "match-with-field") } ) public Match( Source source, - @Param(name = "field", type = { "keyword", "text" }, description = "Field that the query will target.") Expression field, + @Param( + name = "field", + type = { "keyword", "text", "boolean", "date", "date_nanos", "double", "integer", "ip", "long", "unsigned_long", "version" }, + description = "Field that the query will target." + ) Expression field, @Param( name = "query", - type = { "keyword", "text" }, - description = "Text you wish to find in the provided field." + type = { "keyword", "boolean", "date", "date_nanos", "double", "integer", "ip", "long", "unsigned_long", "version" }, + description = "Value to find in the provided field." ) Expression matchQuery ) { super(source, matchQuery, List.of(field, matchQuery)); @@ -84,12 +134,56 @@ public String getWriteableName() { @Override protected TypeResolution resolveNonQueryParamTypes() { - return isNotNull(field, sourceText(), FIRST).and(isString(field, sourceText(), FIRST)).and(super.resolveNonQueryParamTypes()); + return isNotNull(field, sourceText(), FIRST).and( + isType( + field, + FIELD_DATA_TYPES::contains, + sourceText(), + FIRST, + "keyword, text, boolean, date, date_nanos, double, integer, ip, long, unsigned_long, version" + ) + ); + } + + @Override + protected TypeResolution resolveQueryParamType() { + return isType( + query(), + QUERY_DATA_TYPES::contains, + sourceText(), + queryParamOrdinal(), + "keyword, boolean, date, date_nanos, double, integer, ip, long, unsigned_long, version" + ).and(isNotNullAndFoldable(query(), sourceText(), queryParamOrdinal())); + } + + @Override + protected TypeResolution checkParamCompatibility() { + DataType fieldType = field().dataType(); + DataType queryType = query().dataType(); + + // Field and query types should match. If the query is a string, then it can match any field type. + if ((fieldType == queryType) || (queryType == KEYWORD)) { + return TypeResolution.TYPE_RESOLVED; + } + + if (fieldType.isNumeric() && queryType.isNumeric()) { + // When doing an unsigned long query, field must be an unsigned long + if ((queryType == UNSIGNED_LONG && fieldType != UNSIGNED_LONG) == false) { + return TypeResolution.TYPE_RESOLVED; + } + } + + return new TypeResolution(formatIncompatibleTypesMessage(fieldType, queryType, sourceText())); } @Override public void validate(Failures failures) { - if (field instanceof FieldAttribute == false) { + Expression fieldExpression = field(); + // Field may be converted to other data type (field_name :: data_type), so we need to check the original field + if (fieldExpression instanceof AbstractConvertFunction convertFunction) { + fieldExpression = convertFunction.field(); + } + if (fieldExpression instanceof FieldAttribute == false) { failures.add( Failure.fail( field, @@ -102,6 +196,32 @@ public void validate(Failures failures) { } } + @Override + public Object queryAsObject() { + Object queryAsObject = query().fold(); + + // Convert BytesRef to string for string-based values + if (queryAsObject instanceof BytesRef bytesRef) { + return switch (query().dataType()) { + case IP -> EsqlDataTypeConverter.ipToString(bytesRef); + case VERSION -> EsqlDataTypeConverter.versionToString(bytesRef); + default -> bytesRef.utf8ToString(); + }; + } + + // Converts specific types to the correct type for the query + if (query().dataType() == DataType.UNSIGNED_LONG) { + return NumericUtils.unsignedLongAsBigInteger((Long) queryAsObject); + } else if (query().dataType() == DataType.DATETIME && queryAsObject instanceof Long) { + // When casting to date and datetime, we get a long back. But Match query needs a date string + return EsqlDataTypeConverter.dateTimeToString((Long) queryAsObject); + } else if (query().dataType() == DATE_NANOS && queryAsObject instanceof Long) { + return EsqlDataTypeConverter.nanoTimeToString((Long) queryAsObject); + } + + return queryAsObject; + } + @Override public Expression replaceChildren(List newChildren) { return new Match(source(), newChildren.get(0), newChildren.get(1)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java index 0d7d15a13dd80..bd79661534b76 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java @@ -32,7 +32,8 @@ public class QueryString extends FullTextFunction { @FunctionInfo( returnType = "boolean", preview = true, - description = "Performs a query string query. Returns true if the provided query string matches the row.", + description = "Performs a <>. " + + "Returns true if the provided query string matches the row.", examples = { @Example(file = "qstr-function", tag = "qstr-with-field") } ) public QueryString( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index 217c6528c9fd6..3e2a21664aa7e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -243,7 +243,7 @@ protected TypeResolution checkCompatibility() { // Unsigned long is only interoperable with other unsigned longs if ((rightType == UNSIGNED_LONG && (false == (leftType == UNSIGNED_LONG || leftType == DataType.NULL))) || (leftType == UNSIGNED_LONG && (false == (rightType == UNSIGNED_LONG || rightType == DataType.NULL)))) { - return new TypeResolution(formatIncompatibleTypesMessage()); + return new TypeResolution(formatIncompatibleTypesMessage(left().dataType(), right().dataType(), sourceText())); } if ((leftType.isNumeric() && rightType.isNumeric()) @@ -254,35 +254,35 @@ protected TypeResolution checkCompatibility() { || DataType.isNull(rightType)) { return TypeResolution.TYPE_RESOLVED; } - return new TypeResolution(formatIncompatibleTypesMessage()); + return new TypeResolution(formatIncompatibleTypesMessage(left().dataType(), right().dataType(), sourceText())); } - public String formatIncompatibleTypesMessage() { - if (left().dataType().equals(UNSIGNED_LONG)) { + public static String formatIncompatibleTypesMessage(DataType leftType, DataType rightType, String sourceText) { + if (leftType.equals(UNSIGNED_LONG)) { return format( null, "first argument of [{}] is [unsigned_long] and second is [{}]. " + "[unsigned_long] can only be operated on together with another [unsigned_long]", - sourceText(), - right().dataType().typeName() + sourceText, + rightType.typeName() ); } - if (right().dataType().equals(UNSIGNED_LONG)) { + if (rightType.equals(UNSIGNED_LONG)) { return format( null, "first argument of [{}] is [{}] and second is [unsigned_long]. " + "[unsigned_long] can only be operated on together with another [unsigned_long]", - sourceText(), - left().dataType().typeName() + sourceText, + leftType.typeName() ); } return format( null, "first argument of [{}] is [{}] so second argument must also be [{}] but was [{}]", - sourceText(), - left().dataType().isNumeric() ? "numeric" : left().dataType().typeName(), - left().dataType().isNumeric() ? "numeric" : left().dataType().typeName(), - right().dataType().typeName() + sourceText, + leftType.isNumeric() ? "numeric" : leftType.typeName(), + leftType.isNumeric() ? "numeric" : leftType.typeName(), + rightType.typeName() ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index 9d02af0efbab0..6fcdd538fdfc8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -31,7 +31,6 @@ import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Queries; import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; -import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.Term; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction; @@ -253,8 +252,6 @@ static boolean canPushToSource(Expression exp, LucenePushdownPredicates lucenePu && Expressions.foldable(cidrMatch.matches()); } else if (exp instanceof SpatialRelatesFunction spatial) { return canPushSpatialFunctionToSource(spatial, lucenePushdownPredicates); - } else if (exp instanceof Match mf) { - return mf.field() instanceof FieldAttribute && DataType.isString(mf.field().dataType()); } else if (exp instanceof Term term) { return term.field() instanceof FieldAttribute && DataType.isString(term.field().dataType()); } else if (exp instanceof FullTextFunction) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index 50493f584fe4c..c5b37fa411f65 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -330,4 +330,4 @@ joinPredicate atn: -[4, 1, 128, 635, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 142, 8, 1, 10, 1, 12, 1, 145, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 153, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 173, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 185, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 192, 8, 5, 10, 5, 12, 5, 195, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 202, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 207, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 215, 8, 5, 10, 5, 12, 5, 218, 9, 5, 1, 6, 1, 6, 3, 6, 222, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 229, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 234, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 245, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 251, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 259, 8, 9, 10, 9, 12, 9, 262, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 272, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 277, 8, 10, 10, 10, 12, 10, 280, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 288, 8, 11, 10, 11, 12, 11, 291, 9, 11, 3, 11, 293, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 307, 8, 15, 10, 15, 12, 15, 310, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 315, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 323, 8, 17, 10, 17, 12, 17, 326, 9, 17, 1, 17, 3, 17, 329, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 334, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 3, 21, 344, 8, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 350, 8, 22, 10, 22, 12, 22, 353, 9, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 363, 8, 24, 10, 24, 12, 24, 366, 9, 24, 1, 24, 3, 24, 369, 8, 24, 1, 24, 1, 24, 3, 24, 373, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 380, 8, 26, 1, 26, 1, 26, 3, 26, 384, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 389, 8, 27, 10, 27, 12, 27, 392, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 397, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 402, 8, 29, 10, 29, 12, 29, 405, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 410, 8, 30, 10, 30, 12, 30, 413, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 418, 8, 31, 10, 31, 12, 31, 421, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 428, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 443, 8, 34, 10, 34, 12, 34, 446, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 454, 8, 34, 10, 34, 12, 34, 457, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 465, 8, 34, 10, 34, 12, 34, 468, 9, 34, 1, 34, 1, 34, 3, 34, 472, 8, 34, 1, 35, 1, 35, 3, 35, 476, 8, 35, 1, 36, 1, 36, 1, 36, 3, 36, 481, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 490, 8, 38, 10, 38, 12, 38, 493, 9, 38, 1, 39, 1, 39, 3, 39, 497, 8, 39, 1, 39, 1, 39, 3, 39, 501, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 513, 8, 42, 10, 42, 12, 42, 516, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 526, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 538, 8, 47, 10, 47, 12, 47, 541, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 551, 8, 50, 1, 51, 3, 51, 554, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 559, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 581, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 587, 8, 58, 10, 58, 12, 58, 590, 9, 58, 3, 58, 592, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 597, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 610, 8, 61, 1, 62, 3, 62, 613, 8, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 3, 63, 622, 8, 63, 1, 64, 1, 64, 1, 64, 1, 64, 5, 64, 628, 8, 64, 10, 64, 12, 64, 631, 9, 64, 1, 65, 1, 65, 1, 65, 0, 4, 2, 10, 18, 20, 66, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 30, 30, 81, 81, 1, 0, 72, 73, 2, 0, 35, 35, 40, 40, 2, 0, 43, 43, 46, 46, 2, 0, 42, 42, 56, 56, 2, 0, 57, 57, 59, 63, 1, 0, 22, 24, 660, 0, 132, 1, 0, 0, 0, 2, 135, 1, 0, 0, 0, 4, 152, 1, 0, 0, 0, 6, 172, 1, 0, 0, 0, 8, 174, 1, 0, 0, 0, 10, 206, 1, 0, 0, 0, 12, 233, 1, 0, 0, 0, 14, 235, 1, 0, 0, 0, 16, 244, 1, 0, 0, 0, 18, 250, 1, 0, 0, 0, 20, 271, 1, 0, 0, 0, 22, 281, 1, 0, 0, 0, 24, 296, 1, 0, 0, 0, 26, 298, 1, 0, 0, 0, 28, 300, 1, 0, 0, 0, 30, 303, 1, 0, 0, 0, 32, 314, 1, 0, 0, 0, 34, 318, 1, 0, 0, 0, 36, 333, 1, 0, 0, 0, 38, 337, 1, 0, 0, 0, 40, 339, 1, 0, 0, 0, 42, 343, 1, 0, 0, 0, 44, 345, 1, 0, 0, 0, 46, 354, 1, 0, 0, 0, 48, 358, 1, 0, 0, 0, 50, 374, 1, 0, 0, 0, 52, 377, 1, 0, 0, 0, 54, 385, 1, 0, 0, 0, 56, 393, 1, 0, 0, 0, 58, 398, 1, 0, 0, 0, 60, 406, 1, 0, 0, 0, 62, 414, 1, 0, 0, 0, 64, 422, 1, 0, 0, 0, 66, 427, 1, 0, 0, 0, 68, 471, 1, 0, 0, 0, 70, 475, 1, 0, 0, 0, 72, 480, 1, 0, 0, 0, 74, 482, 1, 0, 0, 0, 76, 485, 1, 0, 0, 0, 78, 494, 1, 0, 0, 0, 80, 502, 1, 0, 0, 0, 82, 505, 1, 0, 0, 0, 84, 508, 1, 0, 0, 0, 86, 517, 1, 0, 0, 0, 88, 521, 1, 0, 0, 0, 90, 527, 1, 0, 0, 0, 92, 531, 1, 0, 0, 0, 94, 534, 1, 0, 0, 0, 96, 542, 1, 0, 0, 0, 98, 546, 1, 0, 0, 0, 100, 550, 1, 0, 0, 0, 102, 553, 1, 0, 0, 0, 104, 558, 1, 0, 0, 0, 106, 562, 1, 0, 0, 0, 108, 564, 1, 0, 0, 0, 110, 566, 1, 0, 0, 0, 112, 569, 1, 0, 0, 0, 114, 573, 1, 0, 0, 0, 116, 576, 1, 0, 0, 0, 118, 596, 1, 0, 0, 0, 120, 600, 1, 0, 0, 0, 122, 605, 1, 0, 0, 0, 124, 612, 1, 0, 0, 0, 126, 618, 1, 0, 0, 0, 128, 623, 1, 0, 0, 0, 130, 632, 1, 0, 0, 0, 132, 133, 3, 2, 1, 0, 133, 134, 5, 0, 0, 1, 134, 1, 1, 0, 0, 0, 135, 136, 6, 1, -1, 0, 136, 137, 3, 4, 2, 0, 137, 143, 1, 0, 0, 0, 138, 139, 10, 1, 0, 0, 139, 140, 5, 29, 0, 0, 140, 142, 3, 6, 3, 0, 141, 138, 1, 0, 0, 0, 142, 145, 1, 0, 0, 0, 143, 141, 1, 0, 0, 0, 143, 144, 1, 0, 0, 0, 144, 3, 1, 0, 0, 0, 145, 143, 1, 0, 0, 0, 146, 153, 3, 110, 55, 0, 147, 153, 3, 34, 17, 0, 148, 153, 3, 28, 14, 0, 149, 153, 3, 114, 57, 0, 150, 151, 4, 2, 1, 0, 151, 153, 3, 48, 24, 0, 152, 146, 1, 0, 0, 0, 152, 147, 1, 0, 0, 0, 152, 148, 1, 0, 0, 0, 152, 149, 1, 0, 0, 0, 152, 150, 1, 0, 0, 0, 153, 5, 1, 0, 0, 0, 154, 173, 3, 50, 25, 0, 155, 173, 3, 8, 4, 0, 156, 173, 3, 80, 40, 0, 157, 173, 3, 74, 37, 0, 158, 173, 3, 52, 26, 0, 159, 173, 3, 76, 38, 0, 160, 173, 3, 82, 41, 0, 161, 173, 3, 84, 42, 0, 162, 173, 3, 88, 44, 0, 163, 173, 3, 90, 45, 0, 164, 173, 3, 116, 58, 0, 165, 173, 3, 92, 46, 0, 166, 167, 4, 3, 2, 0, 167, 173, 3, 122, 61, 0, 168, 169, 4, 3, 3, 0, 169, 173, 3, 120, 60, 0, 170, 171, 4, 3, 4, 0, 171, 173, 3, 124, 62, 0, 172, 154, 1, 0, 0, 0, 172, 155, 1, 0, 0, 0, 172, 156, 1, 0, 0, 0, 172, 157, 1, 0, 0, 0, 172, 158, 1, 0, 0, 0, 172, 159, 1, 0, 0, 0, 172, 160, 1, 0, 0, 0, 172, 161, 1, 0, 0, 0, 172, 162, 1, 0, 0, 0, 172, 163, 1, 0, 0, 0, 172, 164, 1, 0, 0, 0, 172, 165, 1, 0, 0, 0, 172, 166, 1, 0, 0, 0, 172, 168, 1, 0, 0, 0, 172, 170, 1, 0, 0, 0, 173, 7, 1, 0, 0, 0, 174, 175, 5, 16, 0, 0, 175, 176, 3, 10, 5, 0, 176, 9, 1, 0, 0, 0, 177, 178, 6, 5, -1, 0, 178, 179, 5, 49, 0, 0, 179, 207, 3, 10, 5, 8, 180, 207, 3, 16, 8, 0, 181, 207, 3, 12, 6, 0, 182, 184, 3, 16, 8, 0, 183, 185, 5, 49, 0, 0, 184, 183, 1, 0, 0, 0, 184, 185, 1, 0, 0, 0, 185, 186, 1, 0, 0, 0, 186, 187, 5, 44, 0, 0, 187, 188, 5, 48, 0, 0, 188, 193, 3, 16, 8, 0, 189, 190, 5, 39, 0, 0, 190, 192, 3, 16, 8, 0, 191, 189, 1, 0, 0, 0, 192, 195, 1, 0, 0, 0, 193, 191, 1, 0, 0, 0, 193, 194, 1, 0, 0, 0, 194, 196, 1, 0, 0, 0, 195, 193, 1, 0, 0, 0, 196, 197, 5, 55, 0, 0, 197, 207, 1, 0, 0, 0, 198, 199, 3, 16, 8, 0, 199, 201, 5, 45, 0, 0, 200, 202, 5, 49, 0, 0, 201, 200, 1, 0, 0, 0, 201, 202, 1, 0, 0, 0, 202, 203, 1, 0, 0, 0, 203, 204, 5, 50, 0, 0, 204, 207, 1, 0, 0, 0, 205, 207, 3, 14, 7, 0, 206, 177, 1, 0, 0, 0, 206, 180, 1, 0, 0, 0, 206, 181, 1, 0, 0, 0, 206, 182, 1, 0, 0, 0, 206, 198, 1, 0, 0, 0, 206, 205, 1, 0, 0, 0, 207, 216, 1, 0, 0, 0, 208, 209, 10, 5, 0, 0, 209, 210, 5, 34, 0, 0, 210, 215, 3, 10, 5, 6, 211, 212, 10, 4, 0, 0, 212, 213, 5, 52, 0, 0, 213, 215, 3, 10, 5, 5, 214, 208, 1, 0, 0, 0, 214, 211, 1, 0, 0, 0, 215, 218, 1, 0, 0, 0, 216, 214, 1, 0, 0, 0, 216, 217, 1, 0, 0, 0, 217, 11, 1, 0, 0, 0, 218, 216, 1, 0, 0, 0, 219, 221, 3, 16, 8, 0, 220, 222, 5, 49, 0, 0, 221, 220, 1, 0, 0, 0, 221, 222, 1, 0, 0, 0, 222, 223, 1, 0, 0, 0, 223, 224, 5, 47, 0, 0, 224, 225, 3, 106, 53, 0, 225, 234, 1, 0, 0, 0, 226, 228, 3, 16, 8, 0, 227, 229, 5, 49, 0, 0, 228, 227, 1, 0, 0, 0, 228, 229, 1, 0, 0, 0, 229, 230, 1, 0, 0, 0, 230, 231, 5, 54, 0, 0, 231, 232, 3, 106, 53, 0, 232, 234, 1, 0, 0, 0, 233, 219, 1, 0, 0, 0, 233, 226, 1, 0, 0, 0, 234, 13, 1, 0, 0, 0, 235, 236, 3, 58, 29, 0, 236, 237, 5, 38, 0, 0, 237, 238, 3, 68, 34, 0, 238, 15, 1, 0, 0, 0, 239, 245, 3, 18, 9, 0, 240, 241, 3, 18, 9, 0, 241, 242, 3, 108, 54, 0, 242, 243, 3, 18, 9, 0, 243, 245, 1, 0, 0, 0, 244, 239, 1, 0, 0, 0, 244, 240, 1, 0, 0, 0, 245, 17, 1, 0, 0, 0, 246, 247, 6, 9, -1, 0, 247, 251, 3, 20, 10, 0, 248, 249, 7, 0, 0, 0, 249, 251, 3, 18, 9, 3, 250, 246, 1, 0, 0, 0, 250, 248, 1, 0, 0, 0, 251, 260, 1, 0, 0, 0, 252, 253, 10, 2, 0, 0, 253, 254, 7, 1, 0, 0, 254, 259, 3, 18, 9, 3, 255, 256, 10, 1, 0, 0, 256, 257, 7, 0, 0, 0, 257, 259, 3, 18, 9, 2, 258, 252, 1, 0, 0, 0, 258, 255, 1, 0, 0, 0, 259, 262, 1, 0, 0, 0, 260, 258, 1, 0, 0, 0, 260, 261, 1, 0, 0, 0, 261, 19, 1, 0, 0, 0, 262, 260, 1, 0, 0, 0, 263, 264, 6, 10, -1, 0, 264, 272, 3, 68, 34, 0, 265, 272, 3, 58, 29, 0, 266, 272, 3, 22, 11, 0, 267, 268, 5, 48, 0, 0, 268, 269, 3, 10, 5, 0, 269, 270, 5, 55, 0, 0, 270, 272, 1, 0, 0, 0, 271, 263, 1, 0, 0, 0, 271, 265, 1, 0, 0, 0, 271, 266, 1, 0, 0, 0, 271, 267, 1, 0, 0, 0, 272, 278, 1, 0, 0, 0, 273, 274, 10, 1, 0, 0, 274, 275, 5, 37, 0, 0, 275, 277, 3, 26, 13, 0, 276, 273, 1, 0, 0, 0, 277, 280, 1, 0, 0, 0, 278, 276, 1, 0, 0, 0, 278, 279, 1, 0, 0, 0, 279, 21, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 281, 282, 3, 24, 12, 0, 282, 292, 5, 48, 0, 0, 283, 293, 5, 66, 0, 0, 284, 289, 3, 10, 5, 0, 285, 286, 5, 39, 0, 0, 286, 288, 3, 10, 5, 0, 287, 285, 1, 0, 0, 0, 288, 291, 1, 0, 0, 0, 289, 287, 1, 0, 0, 0, 289, 290, 1, 0, 0, 0, 290, 293, 1, 0, 0, 0, 291, 289, 1, 0, 0, 0, 292, 283, 1, 0, 0, 0, 292, 284, 1, 0, 0, 0, 292, 293, 1, 0, 0, 0, 293, 294, 1, 0, 0, 0, 294, 295, 5, 55, 0, 0, 295, 23, 1, 0, 0, 0, 296, 297, 3, 72, 36, 0, 297, 25, 1, 0, 0, 0, 298, 299, 3, 64, 32, 0, 299, 27, 1, 0, 0, 0, 300, 301, 5, 12, 0, 0, 301, 302, 3, 30, 15, 0, 302, 29, 1, 0, 0, 0, 303, 308, 3, 32, 16, 0, 304, 305, 5, 39, 0, 0, 305, 307, 3, 32, 16, 0, 306, 304, 1, 0, 0, 0, 307, 310, 1, 0, 0, 0, 308, 306, 1, 0, 0, 0, 308, 309, 1, 0, 0, 0, 309, 31, 1, 0, 0, 0, 310, 308, 1, 0, 0, 0, 311, 312, 3, 58, 29, 0, 312, 313, 5, 36, 0, 0, 313, 315, 1, 0, 0, 0, 314, 311, 1, 0, 0, 0, 314, 315, 1, 0, 0, 0, 315, 316, 1, 0, 0, 0, 316, 317, 3, 10, 5, 0, 317, 33, 1, 0, 0, 0, 318, 319, 5, 6, 0, 0, 319, 324, 3, 36, 18, 0, 320, 321, 5, 39, 0, 0, 321, 323, 3, 36, 18, 0, 322, 320, 1, 0, 0, 0, 323, 326, 1, 0, 0, 0, 324, 322, 1, 0, 0, 0, 324, 325, 1, 0, 0, 0, 325, 328, 1, 0, 0, 0, 326, 324, 1, 0, 0, 0, 327, 329, 3, 42, 21, 0, 328, 327, 1, 0, 0, 0, 328, 329, 1, 0, 0, 0, 329, 35, 1, 0, 0, 0, 330, 331, 3, 38, 19, 0, 331, 332, 5, 38, 0, 0, 332, 334, 1, 0, 0, 0, 333, 330, 1, 0, 0, 0, 333, 334, 1, 0, 0, 0, 334, 335, 1, 0, 0, 0, 335, 336, 3, 40, 20, 0, 336, 37, 1, 0, 0, 0, 337, 338, 5, 81, 0, 0, 338, 39, 1, 0, 0, 0, 339, 340, 7, 2, 0, 0, 340, 41, 1, 0, 0, 0, 341, 344, 3, 44, 22, 0, 342, 344, 3, 46, 23, 0, 343, 341, 1, 0, 0, 0, 343, 342, 1, 0, 0, 0, 344, 43, 1, 0, 0, 0, 345, 346, 5, 80, 0, 0, 346, 351, 5, 81, 0, 0, 347, 348, 5, 39, 0, 0, 348, 350, 5, 81, 0, 0, 349, 347, 1, 0, 0, 0, 350, 353, 1, 0, 0, 0, 351, 349, 1, 0, 0, 0, 351, 352, 1, 0, 0, 0, 352, 45, 1, 0, 0, 0, 353, 351, 1, 0, 0, 0, 354, 355, 5, 70, 0, 0, 355, 356, 3, 44, 22, 0, 356, 357, 5, 71, 0, 0, 357, 47, 1, 0, 0, 0, 358, 359, 5, 19, 0, 0, 359, 364, 3, 36, 18, 0, 360, 361, 5, 39, 0, 0, 361, 363, 3, 36, 18, 0, 362, 360, 1, 0, 0, 0, 363, 366, 1, 0, 0, 0, 364, 362, 1, 0, 0, 0, 364, 365, 1, 0, 0, 0, 365, 368, 1, 0, 0, 0, 366, 364, 1, 0, 0, 0, 367, 369, 3, 54, 27, 0, 368, 367, 1, 0, 0, 0, 368, 369, 1, 0, 0, 0, 369, 372, 1, 0, 0, 0, 370, 371, 5, 33, 0, 0, 371, 373, 3, 30, 15, 0, 372, 370, 1, 0, 0, 0, 372, 373, 1, 0, 0, 0, 373, 49, 1, 0, 0, 0, 374, 375, 5, 4, 0, 0, 375, 376, 3, 30, 15, 0, 376, 51, 1, 0, 0, 0, 377, 379, 5, 15, 0, 0, 378, 380, 3, 54, 27, 0, 379, 378, 1, 0, 0, 0, 379, 380, 1, 0, 0, 0, 380, 383, 1, 0, 0, 0, 381, 382, 5, 33, 0, 0, 382, 384, 3, 30, 15, 0, 383, 381, 1, 0, 0, 0, 383, 384, 1, 0, 0, 0, 384, 53, 1, 0, 0, 0, 385, 390, 3, 56, 28, 0, 386, 387, 5, 39, 0, 0, 387, 389, 3, 56, 28, 0, 388, 386, 1, 0, 0, 0, 389, 392, 1, 0, 0, 0, 390, 388, 1, 0, 0, 0, 390, 391, 1, 0, 0, 0, 391, 55, 1, 0, 0, 0, 392, 390, 1, 0, 0, 0, 393, 396, 3, 32, 16, 0, 394, 395, 5, 16, 0, 0, 395, 397, 3, 10, 5, 0, 396, 394, 1, 0, 0, 0, 396, 397, 1, 0, 0, 0, 397, 57, 1, 0, 0, 0, 398, 403, 3, 72, 36, 0, 399, 400, 5, 41, 0, 0, 400, 402, 3, 72, 36, 0, 401, 399, 1, 0, 0, 0, 402, 405, 1, 0, 0, 0, 403, 401, 1, 0, 0, 0, 403, 404, 1, 0, 0, 0, 404, 59, 1, 0, 0, 0, 405, 403, 1, 0, 0, 0, 406, 411, 3, 66, 33, 0, 407, 408, 5, 41, 0, 0, 408, 410, 3, 66, 33, 0, 409, 407, 1, 0, 0, 0, 410, 413, 1, 0, 0, 0, 411, 409, 1, 0, 0, 0, 411, 412, 1, 0, 0, 0, 412, 61, 1, 0, 0, 0, 413, 411, 1, 0, 0, 0, 414, 419, 3, 60, 30, 0, 415, 416, 5, 39, 0, 0, 416, 418, 3, 60, 30, 0, 417, 415, 1, 0, 0, 0, 418, 421, 1, 0, 0, 0, 419, 417, 1, 0, 0, 0, 419, 420, 1, 0, 0, 0, 420, 63, 1, 0, 0, 0, 421, 419, 1, 0, 0, 0, 422, 423, 7, 3, 0, 0, 423, 65, 1, 0, 0, 0, 424, 428, 5, 85, 0, 0, 425, 426, 4, 33, 10, 0, 426, 428, 3, 70, 35, 0, 427, 424, 1, 0, 0, 0, 427, 425, 1, 0, 0, 0, 428, 67, 1, 0, 0, 0, 429, 472, 5, 50, 0, 0, 430, 431, 3, 104, 52, 0, 431, 432, 5, 72, 0, 0, 432, 472, 1, 0, 0, 0, 433, 472, 3, 102, 51, 0, 434, 472, 3, 104, 52, 0, 435, 472, 3, 98, 49, 0, 436, 472, 3, 70, 35, 0, 437, 472, 3, 106, 53, 0, 438, 439, 5, 70, 0, 0, 439, 444, 3, 100, 50, 0, 440, 441, 5, 39, 0, 0, 441, 443, 3, 100, 50, 0, 442, 440, 1, 0, 0, 0, 443, 446, 1, 0, 0, 0, 444, 442, 1, 0, 0, 0, 444, 445, 1, 0, 0, 0, 445, 447, 1, 0, 0, 0, 446, 444, 1, 0, 0, 0, 447, 448, 5, 71, 0, 0, 448, 472, 1, 0, 0, 0, 449, 450, 5, 70, 0, 0, 450, 455, 3, 98, 49, 0, 451, 452, 5, 39, 0, 0, 452, 454, 3, 98, 49, 0, 453, 451, 1, 0, 0, 0, 454, 457, 1, 0, 0, 0, 455, 453, 1, 0, 0, 0, 455, 456, 1, 0, 0, 0, 456, 458, 1, 0, 0, 0, 457, 455, 1, 0, 0, 0, 458, 459, 5, 71, 0, 0, 459, 472, 1, 0, 0, 0, 460, 461, 5, 70, 0, 0, 461, 466, 3, 106, 53, 0, 462, 463, 5, 39, 0, 0, 463, 465, 3, 106, 53, 0, 464, 462, 1, 0, 0, 0, 465, 468, 1, 0, 0, 0, 466, 464, 1, 0, 0, 0, 466, 467, 1, 0, 0, 0, 467, 469, 1, 0, 0, 0, 468, 466, 1, 0, 0, 0, 469, 470, 5, 71, 0, 0, 470, 472, 1, 0, 0, 0, 471, 429, 1, 0, 0, 0, 471, 430, 1, 0, 0, 0, 471, 433, 1, 0, 0, 0, 471, 434, 1, 0, 0, 0, 471, 435, 1, 0, 0, 0, 471, 436, 1, 0, 0, 0, 471, 437, 1, 0, 0, 0, 471, 438, 1, 0, 0, 0, 471, 449, 1, 0, 0, 0, 471, 460, 1, 0, 0, 0, 472, 69, 1, 0, 0, 0, 473, 476, 5, 53, 0, 0, 474, 476, 5, 69, 0, 0, 475, 473, 1, 0, 0, 0, 475, 474, 1, 0, 0, 0, 476, 71, 1, 0, 0, 0, 477, 481, 3, 64, 32, 0, 478, 479, 4, 36, 11, 0, 479, 481, 3, 70, 35, 0, 480, 477, 1, 0, 0, 0, 480, 478, 1, 0, 0, 0, 481, 73, 1, 0, 0, 0, 482, 483, 5, 9, 0, 0, 483, 484, 5, 31, 0, 0, 484, 75, 1, 0, 0, 0, 485, 486, 5, 14, 0, 0, 486, 491, 3, 78, 39, 0, 487, 488, 5, 39, 0, 0, 488, 490, 3, 78, 39, 0, 489, 487, 1, 0, 0, 0, 490, 493, 1, 0, 0, 0, 491, 489, 1, 0, 0, 0, 491, 492, 1, 0, 0, 0, 492, 77, 1, 0, 0, 0, 493, 491, 1, 0, 0, 0, 494, 496, 3, 10, 5, 0, 495, 497, 7, 4, 0, 0, 496, 495, 1, 0, 0, 0, 496, 497, 1, 0, 0, 0, 497, 500, 1, 0, 0, 0, 498, 499, 5, 51, 0, 0, 499, 501, 7, 5, 0, 0, 500, 498, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 79, 1, 0, 0, 0, 502, 503, 5, 8, 0, 0, 503, 504, 3, 62, 31, 0, 504, 81, 1, 0, 0, 0, 505, 506, 5, 2, 0, 0, 506, 507, 3, 62, 31, 0, 507, 83, 1, 0, 0, 0, 508, 509, 5, 11, 0, 0, 509, 514, 3, 86, 43, 0, 510, 511, 5, 39, 0, 0, 511, 513, 3, 86, 43, 0, 512, 510, 1, 0, 0, 0, 513, 516, 1, 0, 0, 0, 514, 512, 1, 0, 0, 0, 514, 515, 1, 0, 0, 0, 515, 85, 1, 0, 0, 0, 516, 514, 1, 0, 0, 0, 517, 518, 3, 60, 30, 0, 518, 519, 5, 89, 0, 0, 519, 520, 3, 60, 30, 0, 520, 87, 1, 0, 0, 0, 521, 522, 5, 1, 0, 0, 522, 523, 3, 20, 10, 0, 523, 525, 3, 106, 53, 0, 524, 526, 3, 94, 47, 0, 525, 524, 1, 0, 0, 0, 525, 526, 1, 0, 0, 0, 526, 89, 1, 0, 0, 0, 527, 528, 5, 7, 0, 0, 528, 529, 3, 20, 10, 0, 529, 530, 3, 106, 53, 0, 530, 91, 1, 0, 0, 0, 531, 532, 5, 10, 0, 0, 532, 533, 3, 58, 29, 0, 533, 93, 1, 0, 0, 0, 534, 539, 3, 96, 48, 0, 535, 536, 5, 39, 0, 0, 536, 538, 3, 96, 48, 0, 537, 535, 1, 0, 0, 0, 538, 541, 1, 0, 0, 0, 539, 537, 1, 0, 0, 0, 539, 540, 1, 0, 0, 0, 540, 95, 1, 0, 0, 0, 541, 539, 1, 0, 0, 0, 542, 543, 3, 64, 32, 0, 543, 544, 5, 36, 0, 0, 544, 545, 3, 68, 34, 0, 545, 97, 1, 0, 0, 0, 546, 547, 7, 6, 0, 0, 547, 99, 1, 0, 0, 0, 548, 551, 3, 102, 51, 0, 549, 551, 3, 104, 52, 0, 550, 548, 1, 0, 0, 0, 550, 549, 1, 0, 0, 0, 551, 101, 1, 0, 0, 0, 552, 554, 7, 0, 0, 0, 553, 552, 1, 0, 0, 0, 553, 554, 1, 0, 0, 0, 554, 555, 1, 0, 0, 0, 555, 556, 5, 32, 0, 0, 556, 103, 1, 0, 0, 0, 557, 559, 7, 0, 0, 0, 558, 557, 1, 0, 0, 0, 558, 559, 1, 0, 0, 0, 559, 560, 1, 0, 0, 0, 560, 561, 5, 31, 0, 0, 561, 105, 1, 0, 0, 0, 562, 563, 5, 30, 0, 0, 563, 107, 1, 0, 0, 0, 564, 565, 7, 7, 0, 0, 565, 109, 1, 0, 0, 0, 566, 567, 5, 5, 0, 0, 567, 568, 3, 112, 56, 0, 568, 111, 1, 0, 0, 0, 569, 570, 5, 70, 0, 0, 570, 571, 3, 2, 1, 0, 571, 572, 5, 71, 0, 0, 572, 113, 1, 0, 0, 0, 573, 574, 5, 13, 0, 0, 574, 575, 5, 105, 0, 0, 575, 115, 1, 0, 0, 0, 576, 577, 5, 3, 0, 0, 577, 580, 5, 95, 0, 0, 578, 579, 5, 93, 0, 0, 579, 581, 3, 60, 30, 0, 580, 578, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 591, 1, 0, 0, 0, 582, 583, 5, 94, 0, 0, 583, 588, 3, 118, 59, 0, 584, 585, 5, 39, 0, 0, 585, 587, 3, 118, 59, 0, 586, 584, 1, 0, 0, 0, 587, 590, 1, 0, 0, 0, 588, 586, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 592, 1, 0, 0, 0, 590, 588, 1, 0, 0, 0, 591, 582, 1, 0, 0, 0, 591, 592, 1, 0, 0, 0, 592, 117, 1, 0, 0, 0, 593, 594, 3, 60, 30, 0, 594, 595, 5, 36, 0, 0, 595, 597, 1, 0, 0, 0, 596, 593, 1, 0, 0, 0, 596, 597, 1, 0, 0, 0, 597, 598, 1, 0, 0, 0, 598, 599, 3, 60, 30, 0, 599, 119, 1, 0, 0, 0, 600, 601, 5, 18, 0, 0, 601, 602, 3, 36, 18, 0, 602, 603, 5, 93, 0, 0, 603, 604, 3, 62, 31, 0, 604, 121, 1, 0, 0, 0, 605, 606, 5, 17, 0, 0, 606, 609, 3, 54, 27, 0, 607, 608, 5, 33, 0, 0, 608, 610, 3, 30, 15, 0, 609, 607, 1, 0, 0, 0, 609, 610, 1, 0, 0, 0, 610, 123, 1, 0, 0, 0, 611, 613, 7, 8, 0, 0, 612, 611, 1, 0, 0, 0, 612, 613, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 615, 5, 20, 0, 0, 615, 616, 3, 126, 63, 0, 616, 617, 3, 128, 64, 0, 617, 125, 1, 0, 0, 0, 618, 621, 3, 64, 32, 0, 619, 620, 5, 89, 0, 0, 620, 622, 3, 64, 32, 0, 621, 619, 1, 0, 0, 0, 621, 622, 1, 0, 0, 0, 622, 127, 1, 0, 0, 0, 623, 624, 5, 93, 0, 0, 624, 629, 3, 130, 65, 0, 625, 626, 5, 39, 0, 0, 626, 628, 3, 130, 65, 0, 627, 625, 1, 0, 0, 0, 628, 631, 1, 0, 0, 0, 629, 627, 1, 0, 0, 0, 629, 630, 1, 0, 0, 0, 630, 129, 1, 0, 0, 0, 631, 629, 1, 0, 0, 0, 632, 633, 3, 16, 8, 0, 633, 131, 1, 0, 0, 0, 61, 143, 152, 172, 184, 193, 201, 206, 214, 216, 221, 228, 233, 244, 250, 258, 260, 271, 278, 289, 292, 308, 314, 324, 328, 333, 343, 351, 364, 368, 372, 379, 383, 390, 396, 403, 411, 419, 427, 444, 455, 466, 471, 475, 480, 491, 496, 500, 514, 525, 539, 550, 553, 558, 580, 588, 591, 596, 609, 612, 621, 629] \ No newline at end of file +[4, 1, 128, 639, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 142, 8, 1, 10, 1, 12, 1, 145, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 153, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 173, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 185, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 192, 8, 5, 10, 5, 12, 5, 195, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 202, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 207, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 215, 8, 5, 10, 5, 12, 5, 218, 9, 5, 1, 6, 1, 6, 3, 6, 222, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 229, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 234, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 239, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 249, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 255, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 263, 8, 9, 10, 9, 12, 9, 266, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 276, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 281, 8, 10, 10, 10, 12, 10, 284, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 292, 8, 11, 10, 11, 12, 11, 295, 9, 11, 3, 11, 297, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 311, 8, 15, 10, 15, 12, 15, 314, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 319, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 327, 8, 17, 10, 17, 12, 17, 330, 9, 17, 1, 17, 3, 17, 333, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 338, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 3, 21, 348, 8, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 354, 8, 22, 10, 22, 12, 22, 357, 9, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 367, 8, 24, 10, 24, 12, 24, 370, 9, 24, 1, 24, 3, 24, 373, 8, 24, 1, 24, 1, 24, 3, 24, 377, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 384, 8, 26, 1, 26, 1, 26, 3, 26, 388, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 393, 8, 27, 10, 27, 12, 27, 396, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 401, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 406, 8, 29, 10, 29, 12, 29, 409, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 414, 8, 30, 10, 30, 12, 30, 417, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 422, 8, 31, 10, 31, 12, 31, 425, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 432, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 447, 8, 34, 10, 34, 12, 34, 450, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 458, 8, 34, 10, 34, 12, 34, 461, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 469, 8, 34, 10, 34, 12, 34, 472, 9, 34, 1, 34, 1, 34, 3, 34, 476, 8, 34, 1, 35, 1, 35, 3, 35, 480, 8, 35, 1, 36, 1, 36, 1, 36, 3, 36, 485, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 494, 8, 38, 10, 38, 12, 38, 497, 9, 38, 1, 39, 1, 39, 3, 39, 501, 8, 39, 1, 39, 1, 39, 3, 39, 505, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 517, 8, 42, 10, 42, 12, 42, 520, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 530, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 542, 8, 47, 10, 47, 12, 47, 545, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 555, 8, 50, 1, 51, 3, 51, 558, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 563, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 585, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 591, 8, 58, 10, 58, 12, 58, 594, 9, 58, 3, 58, 596, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 601, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 614, 8, 61, 1, 62, 3, 62, 617, 8, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 3, 63, 626, 8, 63, 1, 64, 1, 64, 1, 64, 1, 64, 5, 64, 632, 8, 64, 10, 64, 12, 64, 635, 9, 64, 1, 65, 1, 65, 1, 65, 0, 4, 2, 10, 18, 20, 66, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 30, 30, 81, 81, 1, 0, 72, 73, 2, 0, 35, 35, 40, 40, 2, 0, 43, 43, 46, 46, 2, 0, 42, 42, 56, 56, 2, 0, 57, 57, 59, 63, 1, 0, 22, 24, 665, 0, 132, 1, 0, 0, 0, 2, 135, 1, 0, 0, 0, 4, 152, 1, 0, 0, 0, 6, 172, 1, 0, 0, 0, 8, 174, 1, 0, 0, 0, 10, 206, 1, 0, 0, 0, 12, 233, 1, 0, 0, 0, 14, 235, 1, 0, 0, 0, 16, 248, 1, 0, 0, 0, 18, 254, 1, 0, 0, 0, 20, 275, 1, 0, 0, 0, 22, 285, 1, 0, 0, 0, 24, 300, 1, 0, 0, 0, 26, 302, 1, 0, 0, 0, 28, 304, 1, 0, 0, 0, 30, 307, 1, 0, 0, 0, 32, 318, 1, 0, 0, 0, 34, 322, 1, 0, 0, 0, 36, 337, 1, 0, 0, 0, 38, 341, 1, 0, 0, 0, 40, 343, 1, 0, 0, 0, 42, 347, 1, 0, 0, 0, 44, 349, 1, 0, 0, 0, 46, 358, 1, 0, 0, 0, 48, 362, 1, 0, 0, 0, 50, 378, 1, 0, 0, 0, 52, 381, 1, 0, 0, 0, 54, 389, 1, 0, 0, 0, 56, 397, 1, 0, 0, 0, 58, 402, 1, 0, 0, 0, 60, 410, 1, 0, 0, 0, 62, 418, 1, 0, 0, 0, 64, 426, 1, 0, 0, 0, 66, 431, 1, 0, 0, 0, 68, 475, 1, 0, 0, 0, 70, 479, 1, 0, 0, 0, 72, 484, 1, 0, 0, 0, 74, 486, 1, 0, 0, 0, 76, 489, 1, 0, 0, 0, 78, 498, 1, 0, 0, 0, 80, 506, 1, 0, 0, 0, 82, 509, 1, 0, 0, 0, 84, 512, 1, 0, 0, 0, 86, 521, 1, 0, 0, 0, 88, 525, 1, 0, 0, 0, 90, 531, 1, 0, 0, 0, 92, 535, 1, 0, 0, 0, 94, 538, 1, 0, 0, 0, 96, 546, 1, 0, 0, 0, 98, 550, 1, 0, 0, 0, 100, 554, 1, 0, 0, 0, 102, 557, 1, 0, 0, 0, 104, 562, 1, 0, 0, 0, 106, 566, 1, 0, 0, 0, 108, 568, 1, 0, 0, 0, 110, 570, 1, 0, 0, 0, 112, 573, 1, 0, 0, 0, 114, 577, 1, 0, 0, 0, 116, 580, 1, 0, 0, 0, 118, 600, 1, 0, 0, 0, 120, 604, 1, 0, 0, 0, 122, 609, 1, 0, 0, 0, 124, 616, 1, 0, 0, 0, 126, 622, 1, 0, 0, 0, 128, 627, 1, 0, 0, 0, 130, 636, 1, 0, 0, 0, 132, 133, 3, 2, 1, 0, 133, 134, 5, 0, 0, 1, 134, 1, 1, 0, 0, 0, 135, 136, 6, 1, -1, 0, 136, 137, 3, 4, 2, 0, 137, 143, 1, 0, 0, 0, 138, 139, 10, 1, 0, 0, 139, 140, 5, 29, 0, 0, 140, 142, 3, 6, 3, 0, 141, 138, 1, 0, 0, 0, 142, 145, 1, 0, 0, 0, 143, 141, 1, 0, 0, 0, 143, 144, 1, 0, 0, 0, 144, 3, 1, 0, 0, 0, 145, 143, 1, 0, 0, 0, 146, 153, 3, 110, 55, 0, 147, 153, 3, 34, 17, 0, 148, 153, 3, 28, 14, 0, 149, 153, 3, 114, 57, 0, 150, 151, 4, 2, 1, 0, 151, 153, 3, 48, 24, 0, 152, 146, 1, 0, 0, 0, 152, 147, 1, 0, 0, 0, 152, 148, 1, 0, 0, 0, 152, 149, 1, 0, 0, 0, 152, 150, 1, 0, 0, 0, 153, 5, 1, 0, 0, 0, 154, 173, 3, 50, 25, 0, 155, 173, 3, 8, 4, 0, 156, 173, 3, 80, 40, 0, 157, 173, 3, 74, 37, 0, 158, 173, 3, 52, 26, 0, 159, 173, 3, 76, 38, 0, 160, 173, 3, 82, 41, 0, 161, 173, 3, 84, 42, 0, 162, 173, 3, 88, 44, 0, 163, 173, 3, 90, 45, 0, 164, 173, 3, 116, 58, 0, 165, 173, 3, 92, 46, 0, 166, 167, 4, 3, 2, 0, 167, 173, 3, 122, 61, 0, 168, 169, 4, 3, 3, 0, 169, 173, 3, 120, 60, 0, 170, 171, 4, 3, 4, 0, 171, 173, 3, 124, 62, 0, 172, 154, 1, 0, 0, 0, 172, 155, 1, 0, 0, 0, 172, 156, 1, 0, 0, 0, 172, 157, 1, 0, 0, 0, 172, 158, 1, 0, 0, 0, 172, 159, 1, 0, 0, 0, 172, 160, 1, 0, 0, 0, 172, 161, 1, 0, 0, 0, 172, 162, 1, 0, 0, 0, 172, 163, 1, 0, 0, 0, 172, 164, 1, 0, 0, 0, 172, 165, 1, 0, 0, 0, 172, 166, 1, 0, 0, 0, 172, 168, 1, 0, 0, 0, 172, 170, 1, 0, 0, 0, 173, 7, 1, 0, 0, 0, 174, 175, 5, 16, 0, 0, 175, 176, 3, 10, 5, 0, 176, 9, 1, 0, 0, 0, 177, 178, 6, 5, -1, 0, 178, 179, 5, 49, 0, 0, 179, 207, 3, 10, 5, 8, 180, 207, 3, 16, 8, 0, 181, 207, 3, 12, 6, 0, 182, 184, 3, 16, 8, 0, 183, 185, 5, 49, 0, 0, 184, 183, 1, 0, 0, 0, 184, 185, 1, 0, 0, 0, 185, 186, 1, 0, 0, 0, 186, 187, 5, 44, 0, 0, 187, 188, 5, 48, 0, 0, 188, 193, 3, 16, 8, 0, 189, 190, 5, 39, 0, 0, 190, 192, 3, 16, 8, 0, 191, 189, 1, 0, 0, 0, 192, 195, 1, 0, 0, 0, 193, 191, 1, 0, 0, 0, 193, 194, 1, 0, 0, 0, 194, 196, 1, 0, 0, 0, 195, 193, 1, 0, 0, 0, 196, 197, 5, 55, 0, 0, 197, 207, 1, 0, 0, 0, 198, 199, 3, 16, 8, 0, 199, 201, 5, 45, 0, 0, 200, 202, 5, 49, 0, 0, 201, 200, 1, 0, 0, 0, 201, 202, 1, 0, 0, 0, 202, 203, 1, 0, 0, 0, 203, 204, 5, 50, 0, 0, 204, 207, 1, 0, 0, 0, 205, 207, 3, 14, 7, 0, 206, 177, 1, 0, 0, 0, 206, 180, 1, 0, 0, 0, 206, 181, 1, 0, 0, 0, 206, 182, 1, 0, 0, 0, 206, 198, 1, 0, 0, 0, 206, 205, 1, 0, 0, 0, 207, 216, 1, 0, 0, 0, 208, 209, 10, 5, 0, 0, 209, 210, 5, 34, 0, 0, 210, 215, 3, 10, 5, 6, 211, 212, 10, 4, 0, 0, 212, 213, 5, 52, 0, 0, 213, 215, 3, 10, 5, 5, 214, 208, 1, 0, 0, 0, 214, 211, 1, 0, 0, 0, 215, 218, 1, 0, 0, 0, 216, 214, 1, 0, 0, 0, 216, 217, 1, 0, 0, 0, 217, 11, 1, 0, 0, 0, 218, 216, 1, 0, 0, 0, 219, 221, 3, 16, 8, 0, 220, 222, 5, 49, 0, 0, 221, 220, 1, 0, 0, 0, 221, 222, 1, 0, 0, 0, 222, 223, 1, 0, 0, 0, 223, 224, 5, 47, 0, 0, 224, 225, 3, 106, 53, 0, 225, 234, 1, 0, 0, 0, 226, 228, 3, 16, 8, 0, 227, 229, 5, 49, 0, 0, 228, 227, 1, 0, 0, 0, 228, 229, 1, 0, 0, 0, 229, 230, 1, 0, 0, 0, 230, 231, 5, 54, 0, 0, 231, 232, 3, 106, 53, 0, 232, 234, 1, 0, 0, 0, 233, 219, 1, 0, 0, 0, 233, 226, 1, 0, 0, 0, 234, 13, 1, 0, 0, 0, 235, 238, 3, 58, 29, 0, 236, 237, 5, 37, 0, 0, 237, 239, 3, 26, 13, 0, 238, 236, 1, 0, 0, 0, 238, 239, 1, 0, 0, 0, 239, 240, 1, 0, 0, 0, 240, 241, 5, 38, 0, 0, 241, 242, 3, 68, 34, 0, 242, 15, 1, 0, 0, 0, 243, 249, 3, 18, 9, 0, 244, 245, 3, 18, 9, 0, 245, 246, 3, 108, 54, 0, 246, 247, 3, 18, 9, 0, 247, 249, 1, 0, 0, 0, 248, 243, 1, 0, 0, 0, 248, 244, 1, 0, 0, 0, 249, 17, 1, 0, 0, 0, 250, 251, 6, 9, -1, 0, 251, 255, 3, 20, 10, 0, 252, 253, 7, 0, 0, 0, 253, 255, 3, 18, 9, 3, 254, 250, 1, 0, 0, 0, 254, 252, 1, 0, 0, 0, 255, 264, 1, 0, 0, 0, 256, 257, 10, 2, 0, 0, 257, 258, 7, 1, 0, 0, 258, 263, 3, 18, 9, 3, 259, 260, 10, 1, 0, 0, 260, 261, 7, 0, 0, 0, 261, 263, 3, 18, 9, 2, 262, 256, 1, 0, 0, 0, 262, 259, 1, 0, 0, 0, 263, 266, 1, 0, 0, 0, 264, 262, 1, 0, 0, 0, 264, 265, 1, 0, 0, 0, 265, 19, 1, 0, 0, 0, 266, 264, 1, 0, 0, 0, 267, 268, 6, 10, -1, 0, 268, 276, 3, 68, 34, 0, 269, 276, 3, 58, 29, 0, 270, 276, 3, 22, 11, 0, 271, 272, 5, 48, 0, 0, 272, 273, 3, 10, 5, 0, 273, 274, 5, 55, 0, 0, 274, 276, 1, 0, 0, 0, 275, 267, 1, 0, 0, 0, 275, 269, 1, 0, 0, 0, 275, 270, 1, 0, 0, 0, 275, 271, 1, 0, 0, 0, 276, 282, 1, 0, 0, 0, 277, 278, 10, 1, 0, 0, 278, 279, 5, 37, 0, 0, 279, 281, 3, 26, 13, 0, 280, 277, 1, 0, 0, 0, 281, 284, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 282, 283, 1, 0, 0, 0, 283, 21, 1, 0, 0, 0, 284, 282, 1, 0, 0, 0, 285, 286, 3, 24, 12, 0, 286, 296, 5, 48, 0, 0, 287, 297, 5, 66, 0, 0, 288, 293, 3, 10, 5, 0, 289, 290, 5, 39, 0, 0, 290, 292, 3, 10, 5, 0, 291, 289, 1, 0, 0, 0, 292, 295, 1, 0, 0, 0, 293, 291, 1, 0, 0, 0, 293, 294, 1, 0, 0, 0, 294, 297, 1, 0, 0, 0, 295, 293, 1, 0, 0, 0, 296, 287, 1, 0, 0, 0, 296, 288, 1, 0, 0, 0, 296, 297, 1, 0, 0, 0, 297, 298, 1, 0, 0, 0, 298, 299, 5, 55, 0, 0, 299, 23, 1, 0, 0, 0, 300, 301, 3, 72, 36, 0, 301, 25, 1, 0, 0, 0, 302, 303, 3, 64, 32, 0, 303, 27, 1, 0, 0, 0, 304, 305, 5, 12, 0, 0, 305, 306, 3, 30, 15, 0, 306, 29, 1, 0, 0, 0, 307, 312, 3, 32, 16, 0, 308, 309, 5, 39, 0, 0, 309, 311, 3, 32, 16, 0, 310, 308, 1, 0, 0, 0, 311, 314, 1, 0, 0, 0, 312, 310, 1, 0, 0, 0, 312, 313, 1, 0, 0, 0, 313, 31, 1, 0, 0, 0, 314, 312, 1, 0, 0, 0, 315, 316, 3, 58, 29, 0, 316, 317, 5, 36, 0, 0, 317, 319, 1, 0, 0, 0, 318, 315, 1, 0, 0, 0, 318, 319, 1, 0, 0, 0, 319, 320, 1, 0, 0, 0, 320, 321, 3, 10, 5, 0, 321, 33, 1, 0, 0, 0, 322, 323, 5, 6, 0, 0, 323, 328, 3, 36, 18, 0, 324, 325, 5, 39, 0, 0, 325, 327, 3, 36, 18, 0, 326, 324, 1, 0, 0, 0, 327, 330, 1, 0, 0, 0, 328, 326, 1, 0, 0, 0, 328, 329, 1, 0, 0, 0, 329, 332, 1, 0, 0, 0, 330, 328, 1, 0, 0, 0, 331, 333, 3, 42, 21, 0, 332, 331, 1, 0, 0, 0, 332, 333, 1, 0, 0, 0, 333, 35, 1, 0, 0, 0, 334, 335, 3, 38, 19, 0, 335, 336, 5, 38, 0, 0, 336, 338, 1, 0, 0, 0, 337, 334, 1, 0, 0, 0, 337, 338, 1, 0, 0, 0, 338, 339, 1, 0, 0, 0, 339, 340, 3, 40, 20, 0, 340, 37, 1, 0, 0, 0, 341, 342, 5, 81, 0, 0, 342, 39, 1, 0, 0, 0, 343, 344, 7, 2, 0, 0, 344, 41, 1, 0, 0, 0, 345, 348, 3, 44, 22, 0, 346, 348, 3, 46, 23, 0, 347, 345, 1, 0, 0, 0, 347, 346, 1, 0, 0, 0, 348, 43, 1, 0, 0, 0, 349, 350, 5, 80, 0, 0, 350, 355, 5, 81, 0, 0, 351, 352, 5, 39, 0, 0, 352, 354, 5, 81, 0, 0, 353, 351, 1, 0, 0, 0, 354, 357, 1, 0, 0, 0, 355, 353, 1, 0, 0, 0, 355, 356, 1, 0, 0, 0, 356, 45, 1, 0, 0, 0, 357, 355, 1, 0, 0, 0, 358, 359, 5, 70, 0, 0, 359, 360, 3, 44, 22, 0, 360, 361, 5, 71, 0, 0, 361, 47, 1, 0, 0, 0, 362, 363, 5, 19, 0, 0, 363, 368, 3, 36, 18, 0, 364, 365, 5, 39, 0, 0, 365, 367, 3, 36, 18, 0, 366, 364, 1, 0, 0, 0, 367, 370, 1, 0, 0, 0, 368, 366, 1, 0, 0, 0, 368, 369, 1, 0, 0, 0, 369, 372, 1, 0, 0, 0, 370, 368, 1, 0, 0, 0, 371, 373, 3, 54, 27, 0, 372, 371, 1, 0, 0, 0, 372, 373, 1, 0, 0, 0, 373, 376, 1, 0, 0, 0, 374, 375, 5, 33, 0, 0, 375, 377, 3, 30, 15, 0, 376, 374, 1, 0, 0, 0, 376, 377, 1, 0, 0, 0, 377, 49, 1, 0, 0, 0, 378, 379, 5, 4, 0, 0, 379, 380, 3, 30, 15, 0, 380, 51, 1, 0, 0, 0, 381, 383, 5, 15, 0, 0, 382, 384, 3, 54, 27, 0, 383, 382, 1, 0, 0, 0, 383, 384, 1, 0, 0, 0, 384, 387, 1, 0, 0, 0, 385, 386, 5, 33, 0, 0, 386, 388, 3, 30, 15, 0, 387, 385, 1, 0, 0, 0, 387, 388, 1, 0, 0, 0, 388, 53, 1, 0, 0, 0, 389, 394, 3, 56, 28, 0, 390, 391, 5, 39, 0, 0, 391, 393, 3, 56, 28, 0, 392, 390, 1, 0, 0, 0, 393, 396, 1, 0, 0, 0, 394, 392, 1, 0, 0, 0, 394, 395, 1, 0, 0, 0, 395, 55, 1, 0, 0, 0, 396, 394, 1, 0, 0, 0, 397, 400, 3, 32, 16, 0, 398, 399, 5, 16, 0, 0, 399, 401, 3, 10, 5, 0, 400, 398, 1, 0, 0, 0, 400, 401, 1, 0, 0, 0, 401, 57, 1, 0, 0, 0, 402, 407, 3, 72, 36, 0, 403, 404, 5, 41, 0, 0, 404, 406, 3, 72, 36, 0, 405, 403, 1, 0, 0, 0, 406, 409, 1, 0, 0, 0, 407, 405, 1, 0, 0, 0, 407, 408, 1, 0, 0, 0, 408, 59, 1, 0, 0, 0, 409, 407, 1, 0, 0, 0, 410, 415, 3, 66, 33, 0, 411, 412, 5, 41, 0, 0, 412, 414, 3, 66, 33, 0, 413, 411, 1, 0, 0, 0, 414, 417, 1, 0, 0, 0, 415, 413, 1, 0, 0, 0, 415, 416, 1, 0, 0, 0, 416, 61, 1, 0, 0, 0, 417, 415, 1, 0, 0, 0, 418, 423, 3, 60, 30, 0, 419, 420, 5, 39, 0, 0, 420, 422, 3, 60, 30, 0, 421, 419, 1, 0, 0, 0, 422, 425, 1, 0, 0, 0, 423, 421, 1, 0, 0, 0, 423, 424, 1, 0, 0, 0, 424, 63, 1, 0, 0, 0, 425, 423, 1, 0, 0, 0, 426, 427, 7, 3, 0, 0, 427, 65, 1, 0, 0, 0, 428, 432, 5, 85, 0, 0, 429, 430, 4, 33, 10, 0, 430, 432, 3, 70, 35, 0, 431, 428, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 432, 67, 1, 0, 0, 0, 433, 476, 5, 50, 0, 0, 434, 435, 3, 104, 52, 0, 435, 436, 5, 72, 0, 0, 436, 476, 1, 0, 0, 0, 437, 476, 3, 102, 51, 0, 438, 476, 3, 104, 52, 0, 439, 476, 3, 98, 49, 0, 440, 476, 3, 70, 35, 0, 441, 476, 3, 106, 53, 0, 442, 443, 5, 70, 0, 0, 443, 448, 3, 100, 50, 0, 444, 445, 5, 39, 0, 0, 445, 447, 3, 100, 50, 0, 446, 444, 1, 0, 0, 0, 447, 450, 1, 0, 0, 0, 448, 446, 1, 0, 0, 0, 448, 449, 1, 0, 0, 0, 449, 451, 1, 0, 0, 0, 450, 448, 1, 0, 0, 0, 451, 452, 5, 71, 0, 0, 452, 476, 1, 0, 0, 0, 453, 454, 5, 70, 0, 0, 454, 459, 3, 98, 49, 0, 455, 456, 5, 39, 0, 0, 456, 458, 3, 98, 49, 0, 457, 455, 1, 0, 0, 0, 458, 461, 1, 0, 0, 0, 459, 457, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 462, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 462, 463, 5, 71, 0, 0, 463, 476, 1, 0, 0, 0, 464, 465, 5, 70, 0, 0, 465, 470, 3, 106, 53, 0, 466, 467, 5, 39, 0, 0, 467, 469, 3, 106, 53, 0, 468, 466, 1, 0, 0, 0, 469, 472, 1, 0, 0, 0, 470, 468, 1, 0, 0, 0, 470, 471, 1, 0, 0, 0, 471, 473, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 473, 474, 5, 71, 0, 0, 474, 476, 1, 0, 0, 0, 475, 433, 1, 0, 0, 0, 475, 434, 1, 0, 0, 0, 475, 437, 1, 0, 0, 0, 475, 438, 1, 0, 0, 0, 475, 439, 1, 0, 0, 0, 475, 440, 1, 0, 0, 0, 475, 441, 1, 0, 0, 0, 475, 442, 1, 0, 0, 0, 475, 453, 1, 0, 0, 0, 475, 464, 1, 0, 0, 0, 476, 69, 1, 0, 0, 0, 477, 480, 5, 53, 0, 0, 478, 480, 5, 69, 0, 0, 479, 477, 1, 0, 0, 0, 479, 478, 1, 0, 0, 0, 480, 71, 1, 0, 0, 0, 481, 485, 3, 64, 32, 0, 482, 483, 4, 36, 11, 0, 483, 485, 3, 70, 35, 0, 484, 481, 1, 0, 0, 0, 484, 482, 1, 0, 0, 0, 485, 73, 1, 0, 0, 0, 486, 487, 5, 9, 0, 0, 487, 488, 5, 31, 0, 0, 488, 75, 1, 0, 0, 0, 489, 490, 5, 14, 0, 0, 490, 495, 3, 78, 39, 0, 491, 492, 5, 39, 0, 0, 492, 494, 3, 78, 39, 0, 493, 491, 1, 0, 0, 0, 494, 497, 1, 0, 0, 0, 495, 493, 1, 0, 0, 0, 495, 496, 1, 0, 0, 0, 496, 77, 1, 0, 0, 0, 497, 495, 1, 0, 0, 0, 498, 500, 3, 10, 5, 0, 499, 501, 7, 4, 0, 0, 500, 499, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 504, 1, 0, 0, 0, 502, 503, 5, 51, 0, 0, 503, 505, 7, 5, 0, 0, 504, 502, 1, 0, 0, 0, 504, 505, 1, 0, 0, 0, 505, 79, 1, 0, 0, 0, 506, 507, 5, 8, 0, 0, 507, 508, 3, 62, 31, 0, 508, 81, 1, 0, 0, 0, 509, 510, 5, 2, 0, 0, 510, 511, 3, 62, 31, 0, 511, 83, 1, 0, 0, 0, 512, 513, 5, 11, 0, 0, 513, 518, 3, 86, 43, 0, 514, 515, 5, 39, 0, 0, 515, 517, 3, 86, 43, 0, 516, 514, 1, 0, 0, 0, 517, 520, 1, 0, 0, 0, 518, 516, 1, 0, 0, 0, 518, 519, 1, 0, 0, 0, 519, 85, 1, 0, 0, 0, 520, 518, 1, 0, 0, 0, 521, 522, 3, 60, 30, 0, 522, 523, 5, 89, 0, 0, 523, 524, 3, 60, 30, 0, 524, 87, 1, 0, 0, 0, 525, 526, 5, 1, 0, 0, 526, 527, 3, 20, 10, 0, 527, 529, 3, 106, 53, 0, 528, 530, 3, 94, 47, 0, 529, 528, 1, 0, 0, 0, 529, 530, 1, 0, 0, 0, 530, 89, 1, 0, 0, 0, 531, 532, 5, 7, 0, 0, 532, 533, 3, 20, 10, 0, 533, 534, 3, 106, 53, 0, 534, 91, 1, 0, 0, 0, 535, 536, 5, 10, 0, 0, 536, 537, 3, 58, 29, 0, 537, 93, 1, 0, 0, 0, 538, 543, 3, 96, 48, 0, 539, 540, 5, 39, 0, 0, 540, 542, 3, 96, 48, 0, 541, 539, 1, 0, 0, 0, 542, 545, 1, 0, 0, 0, 543, 541, 1, 0, 0, 0, 543, 544, 1, 0, 0, 0, 544, 95, 1, 0, 0, 0, 545, 543, 1, 0, 0, 0, 546, 547, 3, 64, 32, 0, 547, 548, 5, 36, 0, 0, 548, 549, 3, 68, 34, 0, 549, 97, 1, 0, 0, 0, 550, 551, 7, 6, 0, 0, 551, 99, 1, 0, 0, 0, 552, 555, 3, 102, 51, 0, 553, 555, 3, 104, 52, 0, 554, 552, 1, 0, 0, 0, 554, 553, 1, 0, 0, 0, 555, 101, 1, 0, 0, 0, 556, 558, 7, 0, 0, 0, 557, 556, 1, 0, 0, 0, 557, 558, 1, 0, 0, 0, 558, 559, 1, 0, 0, 0, 559, 560, 5, 32, 0, 0, 560, 103, 1, 0, 0, 0, 561, 563, 7, 0, 0, 0, 562, 561, 1, 0, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 1, 0, 0, 0, 564, 565, 5, 31, 0, 0, 565, 105, 1, 0, 0, 0, 566, 567, 5, 30, 0, 0, 567, 107, 1, 0, 0, 0, 568, 569, 7, 7, 0, 0, 569, 109, 1, 0, 0, 0, 570, 571, 5, 5, 0, 0, 571, 572, 3, 112, 56, 0, 572, 111, 1, 0, 0, 0, 573, 574, 5, 70, 0, 0, 574, 575, 3, 2, 1, 0, 575, 576, 5, 71, 0, 0, 576, 113, 1, 0, 0, 0, 577, 578, 5, 13, 0, 0, 578, 579, 5, 105, 0, 0, 579, 115, 1, 0, 0, 0, 580, 581, 5, 3, 0, 0, 581, 584, 5, 95, 0, 0, 582, 583, 5, 93, 0, 0, 583, 585, 3, 60, 30, 0, 584, 582, 1, 0, 0, 0, 584, 585, 1, 0, 0, 0, 585, 595, 1, 0, 0, 0, 586, 587, 5, 94, 0, 0, 587, 592, 3, 118, 59, 0, 588, 589, 5, 39, 0, 0, 589, 591, 3, 118, 59, 0, 590, 588, 1, 0, 0, 0, 591, 594, 1, 0, 0, 0, 592, 590, 1, 0, 0, 0, 592, 593, 1, 0, 0, 0, 593, 596, 1, 0, 0, 0, 594, 592, 1, 0, 0, 0, 595, 586, 1, 0, 0, 0, 595, 596, 1, 0, 0, 0, 596, 117, 1, 0, 0, 0, 597, 598, 3, 60, 30, 0, 598, 599, 5, 36, 0, 0, 599, 601, 1, 0, 0, 0, 600, 597, 1, 0, 0, 0, 600, 601, 1, 0, 0, 0, 601, 602, 1, 0, 0, 0, 602, 603, 3, 60, 30, 0, 603, 119, 1, 0, 0, 0, 604, 605, 5, 18, 0, 0, 605, 606, 3, 36, 18, 0, 606, 607, 5, 93, 0, 0, 607, 608, 3, 62, 31, 0, 608, 121, 1, 0, 0, 0, 609, 610, 5, 17, 0, 0, 610, 613, 3, 54, 27, 0, 611, 612, 5, 33, 0, 0, 612, 614, 3, 30, 15, 0, 613, 611, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 123, 1, 0, 0, 0, 615, 617, 7, 8, 0, 0, 616, 615, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 619, 5, 20, 0, 0, 619, 620, 3, 126, 63, 0, 620, 621, 3, 128, 64, 0, 621, 125, 1, 0, 0, 0, 622, 625, 3, 64, 32, 0, 623, 624, 5, 89, 0, 0, 624, 626, 3, 64, 32, 0, 625, 623, 1, 0, 0, 0, 625, 626, 1, 0, 0, 0, 626, 127, 1, 0, 0, 0, 627, 628, 5, 93, 0, 0, 628, 633, 3, 130, 65, 0, 629, 630, 5, 39, 0, 0, 630, 632, 3, 130, 65, 0, 631, 629, 1, 0, 0, 0, 632, 635, 1, 0, 0, 0, 633, 631, 1, 0, 0, 0, 633, 634, 1, 0, 0, 0, 634, 129, 1, 0, 0, 0, 635, 633, 1, 0, 0, 0, 636, 637, 3, 16, 8, 0, 637, 131, 1, 0, 0, 0, 62, 143, 152, 172, 184, 193, 201, 206, 214, 216, 221, 228, 233, 238, 248, 254, 262, 264, 275, 282, 293, 296, 312, 318, 328, 332, 337, 347, 355, 368, 372, 376, 383, 387, 394, 400, 407, 415, 423, 431, 448, 459, 470, 475, 479, 484, 495, 500, 504, 518, 529, 543, 554, 557, 562, 584, 592, 595, 600, 613, 616, 625, 633] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index e864eaff3edd7..a56035364641d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -1173,7 +1173,8 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog @SuppressWarnings("CheckReturnValue") public static class MatchBooleanExpressionContext extends ParserRuleContext { public QualifiedNameContext fieldExp; - public ConstantContext queryString; + public DataTypeContext fieldType; + public ConstantContext matchQuery; public TerminalNode COLON() { return getToken(EsqlBaseParser.COLON, 0); } public QualifiedNameContext qualifiedName() { return getRuleContext(QualifiedNameContext.class,0); @@ -1181,6 +1182,10 @@ public QualifiedNameContext qualifiedName() { public ConstantContext constant() { return getRuleContext(ConstantContext.class,0); } + public TerminalNode CAST_OP() { return getToken(EsqlBaseParser.CAST_OP, 0); } + public DataTypeContext dataType() { + return getRuleContext(DataTypeContext.class,0); + } @SuppressWarnings("this-escape") public MatchBooleanExpressionContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -1204,15 +1209,28 @@ public T accept(ParseTreeVisitor visitor) { public final MatchBooleanExpressionContext matchBooleanExpression() throws RecognitionException { MatchBooleanExpressionContext _localctx = new MatchBooleanExpressionContext(_ctx, getState()); enterRule(_localctx, 14, RULE_matchBooleanExpression); + int _la; try { enterOuterAlt(_localctx, 1); { setState(235); ((MatchBooleanExpressionContext)_localctx).fieldExp = qualifiedName(); - setState(236); + setState(238); + _errHandler.sync(this); + _la = _input.LA(1); + if (_la==CAST_OP) { + { + setState(236); + match(CAST_OP); + setState(237); + ((MatchBooleanExpressionContext)_localctx).fieldType = dataType(); + } + } + + setState(240); match(COLON); - setState(237); - ((MatchBooleanExpressionContext)_localctx).queryString = constant(); + setState(241); + ((MatchBooleanExpressionContext)_localctx).matchQuery = constant(); } } catch (RecognitionException re) { @@ -1295,14 +1313,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState()); enterRule(_localctx, 16, RULE_valueExpression); try { - setState(244); + setState(248); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: _localctx = new ValueExpressionDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(239); + setState(243); operatorExpression(0); } break; @@ -1310,11 +1328,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio _localctx = new ComparisonContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(240); + setState(244); ((ComparisonContext)_localctx).left = operatorExpression(0); - setState(241); + setState(245); comparisonOperator(); - setState(242); + setState(246); ((ComparisonContext)_localctx).right = operatorExpression(0); } break; @@ -1439,16 +1457,16 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE int _alt; enterOuterAlt(_localctx, 1); { - setState(250); + setState(254); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: { _localctx = new OperatorExpressionDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(247); + setState(251); primaryExpression(0); } break; @@ -1457,7 +1475,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(248); + setState(252); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1468,31 +1486,31 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(249); + setState(253); operatorExpression(3); } break; } _ctx.stop = _input.LT(-1); - setState(260); + setState(264); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,15,_ctx); + _alt = getInterpreter().adaptivePredict(_input,16,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(258); + setState(262); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) { case 1: { _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(252); + setState(256); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(253); + setState(257); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(((((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & 7L) != 0)) ) { @@ -1503,7 +1521,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(254); + setState(258); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3); } break; @@ -1512,9 +1530,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(255); + setState(259); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(256); + setState(260); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1525,16 +1543,16 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(257); + setState(261); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2); } break; } } } - setState(262); + setState(266); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,15,_ctx); + _alt = getInterpreter().adaptivePredict(_input,16,_ctx); } } } @@ -1690,16 +1708,16 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(271); + setState(275); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,17,_ctx) ) { case 1: { _localctx = new ConstantDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(264); + setState(268); constant(); } break; @@ -1708,7 +1726,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new DereferenceContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(265); + setState(269); qualifiedName(); } break; @@ -1717,7 +1735,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new FunctionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(266); + setState(270); functionExpression(); } break; @@ -1726,19 +1744,19 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new ParenthesizedExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(267); + setState(271); match(LP); - setState(268); + setState(272); booleanExpression(0); - setState(269); + setState(273); match(RP); } break; } _ctx.stop = _input.LT(-1); - setState(278); + setState(282); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,17,_ctx); + _alt = getInterpreter().adaptivePredict(_input,18,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { if ( _parseListeners!=null ) triggerExitRuleEvent(); @@ -1747,18 +1765,18 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc { _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression); - setState(273); + setState(277); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(274); + setState(278); match(CAST_OP); - setState(275); + setState(279); dataType(); } } } - setState(280); + setState(284); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,17,_ctx); + _alt = getInterpreter().adaptivePredict(_input,18,_ctx); } } } @@ -1818,37 +1836,37 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(281); + setState(285); functionName(); - setState(282); + setState(286); match(LP); - setState(292); + setState(296); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) { case 1: { - setState(283); + setState(287); match(ASTERISK); } break; case 2: { { - setState(284); + setState(288); booleanExpression(0); - setState(289); + setState(293); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(285); + setState(289); match(COMMA); - setState(286); + setState(290); booleanExpression(0); } } - setState(291); + setState(295); _errHandler.sync(this); _la = _input.LA(1); } @@ -1856,7 +1874,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx } break; } - setState(294); + setState(298); match(RP); } } @@ -1902,7 +1920,7 @@ public final FunctionNameContext functionName() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(296); + setState(300); identifierOrParameter(); } } @@ -1960,7 +1978,7 @@ public final DataTypeContext dataType() throws RecognitionException { _localctx = new ToDataTypeContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(298); + setState(302); identifier(); } } @@ -2007,9 +2025,9 @@ public final RowCommandContext rowCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(300); + setState(304); match(ROW); - setState(301); + setState(305); fields(); } } @@ -2063,25 +2081,25 @@ public final FieldsContext fields() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(303); + setState(307); field(); - setState(308); + setState(312); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,20,_ctx); + _alt = getInterpreter().adaptivePredict(_input,21,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(304); + setState(308); match(COMMA); - setState(305); + setState(309); field(); } } } - setState(310); + setState(314); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,20,_ctx); + _alt = getInterpreter().adaptivePredict(_input,21,_ctx); } } } @@ -2131,19 +2149,19 @@ public final FieldContext field() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(314); + setState(318); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) { case 1: { - setState(311); + setState(315); qualifiedName(); - setState(312); + setState(316); match(ASSIGN); } break; } - setState(316); + setState(320); booleanExpression(0); } } @@ -2201,34 +2219,34 @@ public final FromCommandContext fromCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(318); + setState(322); match(FROM); - setState(319); + setState(323); indexPattern(); - setState(324); + setState(328); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,22,_ctx); + _alt = getInterpreter().adaptivePredict(_input,23,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(320); + setState(324); match(COMMA); - setState(321); + setState(325); indexPattern(); } } } - setState(326); + setState(330); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,22,_ctx); + _alt = getInterpreter().adaptivePredict(_input,23,_ctx); } - setState(328); + setState(332); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { case 1: { - setState(327); + setState(331); metadata(); } break; @@ -2281,19 +2299,19 @@ public final IndexPatternContext indexPattern() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(333); + setState(337); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) { case 1: { - setState(330); + setState(334); clusterString(); - setState(331); + setState(335); match(COLON); } break; } - setState(335); + setState(339); indexString(); } } @@ -2337,7 +2355,7 @@ public final ClusterStringContext clusterString() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(337); + setState(341); match(UNQUOTED_SOURCE); } } @@ -2383,7 +2401,7 @@ public final IndexStringContext indexString() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(339); + setState(343); _la = _input.LA(1); if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) { _errHandler.recoverInline(this); @@ -2438,20 +2456,20 @@ public final MetadataContext metadata() throws RecognitionException { MetadataContext _localctx = new MetadataContext(_ctx, getState()); enterRule(_localctx, 42, RULE_metadata); try { - setState(343); + setState(347); _errHandler.sync(this); switch (_input.LA(1)) { case METADATA: enterOuterAlt(_localctx, 1); { - setState(341); + setState(345); metadataOption(); } break; case OPENING_BRACKET: enterOuterAlt(_localctx, 2); { - setState(342); + setState(346); deprecated_metadata(); } break; @@ -2508,27 +2526,27 @@ public final MetadataOptionContext metadataOption() throws RecognitionException int _alt; enterOuterAlt(_localctx, 1); { - setState(345); + setState(349); match(METADATA); - setState(346); + setState(350); match(UNQUOTED_SOURCE); - setState(351); + setState(355); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,26,_ctx); + _alt = getInterpreter().adaptivePredict(_input,27,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(347); + setState(351); match(COMMA); - setState(348); + setState(352); match(UNQUOTED_SOURCE); } } } - setState(353); + setState(357); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,26,_ctx); + _alt = getInterpreter().adaptivePredict(_input,27,_ctx); } } } @@ -2575,11 +2593,11 @@ public final Deprecated_metadataContext deprecated_metadata() throws Recognition try { enterOuterAlt(_localctx, 1); { - setState(354); + setState(358); match(OPENING_BRACKET); - setState(355); + setState(359); metadataOption(); - setState(356); + setState(360); match(CLOSING_BRACKET); } } @@ -2643,46 +2661,46 @@ public final MetricsCommandContext metricsCommand() throws RecognitionException int _alt; enterOuterAlt(_localctx, 1); { - setState(358); + setState(362); match(DEV_METRICS); - setState(359); + setState(363); indexPattern(); - setState(364); + setState(368); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,27,_ctx); + _alt = getInterpreter().adaptivePredict(_input,28,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(360); + setState(364); match(COMMA); - setState(361); + setState(365); indexPattern(); } } } - setState(366); + setState(370); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,27,_ctx); + _alt = getInterpreter().adaptivePredict(_input,28,_ctx); } - setState(368); + setState(372); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { case 1: { - setState(367); + setState(371); ((MetricsCommandContext)_localctx).aggregates = aggFields(); } break; } - setState(372); + setState(376); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { case 1: { - setState(370); + setState(374); match(BY); - setState(371); + setState(375); ((MetricsCommandContext)_localctx).grouping = fields(); } break; @@ -2732,9 +2750,9 @@ public final EvalCommandContext evalCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(374); + setState(378); match(EVAL); - setState(375); + setState(379); fields(); } } @@ -2787,26 +2805,26 @@ public final StatsCommandContext statsCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(377); + setState(381); match(STATS); - setState(379); + setState(383); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { case 1: { - setState(378); + setState(382); ((StatsCommandContext)_localctx).stats = aggFields(); } break; } - setState(383); + setState(387); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) { case 1: { - setState(381); + setState(385); match(BY); - setState(382); + setState(386); ((StatsCommandContext)_localctx).grouping = fields(); } break; @@ -2863,25 +2881,25 @@ public final AggFieldsContext aggFields() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(385); + setState(389); aggField(); - setState(390); + setState(394); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,32,_ctx); + _alt = getInterpreter().adaptivePredict(_input,33,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(386); + setState(390); match(COMMA); - setState(387); + setState(391); aggField(); } } } - setState(392); + setState(396); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,32,_ctx); + _alt = getInterpreter().adaptivePredict(_input,33,_ctx); } } } @@ -2931,16 +2949,16 @@ public final AggFieldContext aggField() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(393); + setState(397); field(); - setState(396); + setState(400); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) { case 1: { - setState(394); + setState(398); match(WHERE); - setState(395); + setState(399); booleanExpression(0); } break; @@ -2997,25 +3015,25 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(398); + setState(402); identifierOrParameter(); - setState(403); + setState(407); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,34,_ctx); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(399); + setState(403); match(DOT); - setState(400); + setState(404); identifierOrParameter(); } } } - setState(405); + setState(409); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,34,_ctx); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); } } } @@ -3069,25 +3087,25 @@ public final QualifiedNamePatternContext qualifiedNamePattern() throws Recogniti int _alt; enterOuterAlt(_localctx, 1); { - setState(406); + setState(410); identifierPattern(); - setState(411); + setState(415); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,35,_ctx); + _alt = getInterpreter().adaptivePredict(_input,36,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(407); + setState(411); match(DOT); - setState(408); + setState(412); identifierPattern(); } } } - setState(413); + setState(417); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,35,_ctx); + _alt = getInterpreter().adaptivePredict(_input,36,_ctx); } } } @@ -3141,25 +3159,25 @@ public final QualifiedNamePatternsContext qualifiedNamePatterns() throws Recogni int _alt; enterOuterAlt(_localctx, 1); { - setState(414); + setState(418); qualifiedNamePattern(); - setState(419); + setState(423); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,36,_ctx); + _alt = getInterpreter().adaptivePredict(_input,37,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(415); + setState(419); match(COMMA); - setState(416); + setState(420); qualifiedNamePattern(); } } } - setState(421); + setState(425); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,36,_ctx); + _alt = getInterpreter().adaptivePredict(_input,37,_ctx); } } } @@ -3205,7 +3223,7 @@ public final IdentifierContext identifier() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(422); + setState(426); _la = _input.LA(1); if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -3258,22 +3276,22 @@ public final IdentifierPatternContext identifierPattern() throws RecognitionExce IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState()); enterRule(_localctx, 66, RULE_identifierPattern); try { - setState(427); + setState(431); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,37,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(424); + setState(428); match(ID_PATTERN); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(425); + setState(429); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(426); + setState(430); parameter(); } break; @@ -3546,14 +3564,14 @@ public final ConstantContext constant() throws RecognitionException { enterRule(_localctx, 68, RULE_constant); int _la; try { - setState(471); + setState(475); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(429); + setState(433); match(NULL); } break; @@ -3561,9 +3579,9 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(430); + setState(434); integerValue(); - setState(431); + setState(435); match(UNQUOTED_IDENTIFIER); } break; @@ -3571,7 +3589,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(433); + setState(437); decimalValue(); } break; @@ -3579,7 +3597,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(434); + setState(438); integerValue(); } break; @@ -3587,7 +3605,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(435); + setState(439); booleanValue(); } break; @@ -3595,7 +3613,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new InputParameterContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(436); + setState(440); parameter(); } break; @@ -3603,7 +3621,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(437); + setState(441); string(); } break; @@ -3611,27 +3629,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(438); + setState(442); match(OPENING_BRACKET); - setState(439); + setState(443); numericValue(); - setState(444); + setState(448); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(440); + setState(444); match(COMMA); - setState(441); + setState(445); numericValue(); } } - setState(446); + setState(450); _errHandler.sync(this); _la = _input.LA(1); } - setState(447); + setState(451); match(CLOSING_BRACKET); } break; @@ -3639,27 +3657,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(449); + setState(453); match(OPENING_BRACKET); - setState(450); + setState(454); booleanValue(); - setState(455); + setState(459); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(451); + setState(455); match(COMMA); - setState(452); + setState(456); booleanValue(); } } - setState(457); + setState(461); _errHandler.sync(this); _la = _input.LA(1); } - setState(458); + setState(462); match(CLOSING_BRACKET); } break; @@ -3667,27 +3685,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(460); + setState(464); match(OPENING_BRACKET); - setState(461); + setState(465); string(); - setState(466); + setState(470); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(462); + setState(466); match(COMMA); - setState(463); + setState(467); string(); } } - setState(468); + setState(472); _errHandler.sync(this); _la = _input.LA(1); } - setState(469); + setState(473); match(CLOSING_BRACKET); } break; @@ -3761,14 +3779,14 @@ public final ParameterContext parameter() throws RecognitionException { ParameterContext _localctx = new ParameterContext(_ctx, getState()); enterRule(_localctx, 70, RULE_parameter); try { - setState(475); + setState(479); _errHandler.sync(this); switch (_input.LA(1)) { case PARAM: _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(473); + setState(477); match(PARAM); } break; @@ -3776,7 +3794,7 @@ public final ParameterContext parameter() throws RecognitionException { _localctx = new InputNamedOrPositionalParamContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(474); + setState(478); match(NAMED_OR_POSITIONAL_PARAM); } break; @@ -3827,22 +3845,22 @@ public final IdentifierOrParameterContext identifierOrParameter() throws Recogni IdentifierOrParameterContext _localctx = new IdentifierOrParameterContext(_ctx, getState()); enterRule(_localctx, 72, RULE_identifierOrParameter); try { - setState(480); + setState(484); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(477); + setState(481); identifier(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(478); + setState(482); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(479); + setState(483); parameter(); } break; @@ -3889,9 +3907,9 @@ public final LimitCommandContext limitCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(482); + setState(486); match(LIMIT); - setState(483); + setState(487); match(INTEGER_LITERAL); } } @@ -3946,27 +3964,27 @@ public final SortCommandContext sortCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(485); + setState(489); match(SORT); - setState(486); + setState(490); orderExpression(); - setState(491); + setState(495); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,44,_ctx); + _alt = getInterpreter().adaptivePredict(_input,45,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(487); + setState(491); match(COMMA); - setState(488); + setState(492); orderExpression(); } } } - setState(493); + setState(497); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,44,_ctx); + _alt = getInterpreter().adaptivePredict(_input,45,_ctx); } } } @@ -4020,14 +4038,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(494); + setState(498); booleanExpression(0); - setState(496); + setState(500); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,45,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { case 1: { - setState(495); + setState(499); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -4041,14 +4059,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio } break; } - setState(500); + setState(504); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { case 1: { - setState(498); + setState(502); match(NULLS); - setState(499); + setState(503); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -4107,9 +4125,9 @@ public final KeepCommandContext keepCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(502); + setState(506); match(KEEP); - setState(503); + setState(507); qualifiedNamePatterns(); } } @@ -4156,9 +4174,9 @@ public final DropCommandContext dropCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(505); + setState(509); match(DROP); - setState(506); + setState(510); qualifiedNamePatterns(); } } @@ -4213,27 +4231,27 @@ public final RenameCommandContext renameCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(508); + setState(512); match(RENAME); - setState(509); + setState(513); renameClause(); - setState(514); + setState(518); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,47,_ctx); + _alt = getInterpreter().adaptivePredict(_input,48,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(510); + setState(514); match(COMMA); - setState(511); + setState(515); renameClause(); } } } - setState(516); + setState(520); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,47,_ctx); + _alt = getInterpreter().adaptivePredict(_input,48,_ctx); } } } @@ -4285,11 +4303,11 @@ public final RenameClauseContext renameClause() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(517); + setState(521); ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); - setState(518); + setState(522); match(AS); - setState(519); + setState(523); ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } @@ -4342,18 +4360,18 @@ public final DissectCommandContext dissectCommand() throws RecognitionException try { enterOuterAlt(_localctx, 1); { - setState(521); + setState(525); match(DISSECT); - setState(522); + setState(526); primaryExpression(0); - setState(523); + setState(527); string(); - setState(525); + setState(529); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { case 1: { - setState(524); + setState(528); commandOptions(); } break; @@ -4406,11 +4424,11 @@ public final GrokCommandContext grokCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(527); + setState(531); match(GROK); - setState(528); + setState(532); primaryExpression(0); - setState(529); + setState(533); string(); } } @@ -4457,9 +4475,9 @@ public final MvExpandCommandContext mvExpandCommand() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(531); + setState(535); match(MV_EXPAND); - setState(532); + setState(536); qualifiedName(); } } @@ -4513,25 +4531,25 @@ public final CommandOptionsContext commandOptions() throws RecognitionException int _alt; enterOuterAlt(_localctx, 1); { - setState(534); + setState(538); commandOption(); - setState(539); + setState(543); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,49,_ctx); + _alt = getInterpreter().adaptivePredict(_input,50,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(535); + setState(539); match(COMMA); - setState(536); + setState(540); commandOption(); } } } - setState(541); + setState(545); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,49,_ctx); + _alt = getInterpreter().adaptivePredict(_input,50,_ctx); } } } @@ -4581,11 +4599,11 @@ public final CommandOptionContext commandOption() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(542); + setState(546); identifier(); - setState(543); + setState(547); match(ASSIGN); - setState(544); + setState(548); constant(); } } @@ -4631,7 +4649,7 @@ public final BooleanValueContext booleanValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(546); + setState(550); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -4686,20 +4704,20 @@ public final NumericValueContext numericValue() throws RecognitionException { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); enterRule(_localctx, 100, RULE_numericValue); try { - setState(550); + setState(554); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(548); + setState(552); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(549); + setState(553); integerValue(); } break; @@ -4748,12 +4766,12 @@ public final DecimalValueContext decimalValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(553); + setState(557); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(552); + setState(556); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4766,7 +4784,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException { } } - setState(555); + setState(559); match(DECIMAL_LITERAL); } } @@ -4813,12 +4831,12 @@ public final IntegerValueContext integerValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(558); + setState(562); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(557); + setState(561); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4831,7 +4849,7 @@ public final IntegerValueContext integerValue() throws RecognitionException { } } - setState(560); + setState(564); match(INTEGER_LITERAL); } } @@ -4875,7 +4893,7 @@ public final StringContext string() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(562); + setState(566); match(QUOTED_STRING); } } @@ -4925,7 +4943,7 @@ public final ComparisonOperatorContext comparisonOperator() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(564); + setState(568); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & -432345564227567616L) != 0)) ) { _errHandler.recoverInline(this); @@ -4980,9 +4998,9 @@ public final ExplainCommandContext explainCommand() throws RecognitionException try { enterOuterAlt(_localctx, 1); { - setState(566); + setState(570); match(EXPLAIN); - setState(567); + setState(571); subqueryExpression(); } } @@ -5030,11 +5048,11 @@ public final SubqueryExpressionContext subqueryExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(569); + setState(573); match(OPENING_BRACKET); - setState(570); + setState(574); query(0); - setState(571); + setState(575); match(CLOSING_BRACKET); } } @@ -5091,9 +5109,9 @@ public final ShowCommandContext showCommand() throws RecognitionException { _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(573); + setState(577); match(SHOW); - setState(574); + setState(578); match(INFO); } } @@ -5156,48 +5174,48 @@ public final EnrichCommandContext enrichCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(576); + setState(580); match(ENRICH); - setState(577); + setState(581); ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME); - setState(580); + setState(584); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,53,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,54,_ctx) ) { case 1: { - setState(578); + setState(582); match(ON); - setState(579); + setState(583); ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(591); + setState(595); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,55,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) { case 1: { - setState(582); + setState(586); match(WITH); - setState(583); + setState(587); enrichWithClause(); - setState(588); + setState(592); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,54,_ctx); + _alt = getInterpreter().adaptivePredict(_input,55,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(584); + setState(588); match(COMMA); - setState(585); + setState(589); enrichWithClause(); } } } - setState(590); + setState(594); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,54,_ctx); + _alt = getInterpreter().adaptivePredict(_input,55,_ctx); } } break; @@ -5252,19 +5270,19 @@ public final EnrichWithClauseContext enrichWithClause() throws RecognitionExcept try { enterOuterAlt(_localctx, 1); { - setState(596); + setState(600); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) { case 1: { - setState(593); + setState(597); ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); - setState(594); + setState(598); match(ASSIGN); } break; } - setState(598); + setState(602); ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } @@ -5317,13 +5335,13 @@ public final LookupCommandContext lookupCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(600); + setState(604); match(DEV_LOOKUP); - setState(601); + setState(605); ((LookupCommandContext)_localctx).tableName = indexPattern(); - setState(602); + setState(606); match(ON); - setState(603); + setState(607); ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns(); } } @@ -5376,18 +5394,18 @@ public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(605); + setState(609); match(DEV_INLINESTATS); - setState(606); + setState(610); ((InlinestatsCommandContext)_localctx).stats = aggFields(); - setState(609); + setState(613); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,58,_ctx) ) { case 1: { - setState(607); + setState(611); match(BY); - setState(608); + setState(612); ((InlinestatsCommandContext)_localctx).grouping = fields(); } break; @@ -5445,12 +5463,12 @@ public final JoinCommandContext joinCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(612); + setState(616); _errHandler.sync(this); _la = _input.LA(1); if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 29360128L) != 0)) { { - setState(611); + setState(615); ((JoinCommandContext)_localctx).type = _input.LT(1); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 29360128L) != 0)) ) { @@ -5464,11 +5482,11 @@ public final JoinCommandContext joinCommand() throws RecognitionException { } } - setState(614); + setState(618); match(DEV_JOIN); - setState(615); + setState(619); joinTarget(); - setState(616); + setState(620); joinCondition(); } } @@ -5521,16 +5539,16 @@ public final JoinTargetContext joinTarget() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(618); + setState(622); ((JoinTargetContext)_localctx).index = identifier(); - setState(621); + setState(625); _errHandler.sync(this); _la = _input.LA(1); if (_la==AS) { { - setState(619); + setState(623); match(AS); - setState(620); + setState(624); ((JoinTargetContext)_localctx).alias = identifier(); } } @@ -5588,27 +5606,27 @@ public final JoinConditionContext joinCondition() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(623); + setState(627); match(ON); - setState(624); + setState(628); joinPredicate(); - setState(629); + setState(633); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,60,_ctx); + _alt = getInterpreter().adaptivePredict(_input,61,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(625); + setState(629); match(COMMA); - setState(626); + setState(630); joinPredicate(); } } } - setState(631); + setState(635); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,60,_ctx); + _alt = getInterpreter().adaptivePredict(_input,61,_ctx); } } } @@ -5654,7 +5672,7 @@ public final JoinPredicateContext joinPredicate() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(632); + setState(636); valueExpression(); } } @@ -5756,7 +5774,7 @@ private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _loca } public static final String _serializedATN = - "\u0004\u0001\u0080\u027b\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ + "\u0004\u0001\u0080\u027f\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ "\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004"+ "\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007"+ "\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b"+ @@ -5790,372 +5808,374 @@ private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _loca "\b\u0005\n\u0005\f\u0005\u00da\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006"+ "\u00de\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006"+ "\u0003\u0006\u00e5\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006"+ - "\u00ea\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b"+ - "\u0001\b\u0001\b\u0001\b\u0001\b\u0003\b\u00f5\b\b\u0001\t\u0001\t\u0001"+ - "\t\u0001\t\u0003\t\u00fb\b\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ - "\t\u0005\t\u0103\b\t\n\t\f\t\u0106\t\t\u0001\n\u0001\n\u0001\n\u0001\n"+ - "\u0001\n\u0001\n\u0001\n\u0001\n\u0003\n\u0110\b\n\u0001\n\u0001\n\u0001"+ - "\n\u0005\n\u0115\b\n\n\n\f\n\u0118\t\n\u0001\u000b\u0001\u000b\u0001\u000b"+ - "\u0001\u000b\u0001\u000b\u0001\u000b\u0005\u000b\u0120\b\u000b\n\u000b"+ - "\f\u000b\u0123\t\u000b\u0003\u000b\u0125\b\u000b\u0001\u000b\u0001\u000b"+ - "\u0001\f\u0001\f\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001"+ - "\u000f\u0001\u000f\u0001\u000f\u0005\u000f\u0133\b\u000f\n\u000f\f\u000f"+ - "\u0136\t\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0003\u0010\u013b\b"+ - "\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001"+ - "\u0011\u0005\u0011\u0143\b\u0011\n\u0011\f\u0011\u0146\t\u0011\u0001\u0011"+ - "\u0003\u0011\u0149\b\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0003\u0012"+ - "\u014e\b\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001\u0014"+ - "\u0001\u0014\u0001\u0015\u0001\u0015\u0003\u0015\u0158\b\u0015\u0001\u0016"+ - "\u0001\u0016\u0001\u0016\u0001\u0016\u0005\u0016\u015e\b\u0016\n\u0016"+ - "\f\u0016\u0161\t\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017"+ - "\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0005\u0018\u016b\b\u0018"+ - "\n\u0018\f\u0018\u016e\t\u0018\u0001\u0018\u0003\u0018\u0171\b\u0018\u0001"+ - "\u0018\u0001\u0018\u0003\u0018\u0175\b\u0018\u0001\u0019\u0001\u0019\u0001"+ - "\u0019\u0001\u001a\u0001\u001a\u0003\u001a\u017c\b\u001a\u0001\u001a\u0001"+ - "\u001a\u0003\u001a\u0180\b\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0005"+ - "\u001b\u0185\b\u001b\n\u001b\f\u001b\u0188\t\u001b\u0001\u001c\u0001\u001c"+ - "\u0001\u001c\u0003\u001c\u018d\b\u001c\u0001\u001d\u0001\u001d\u0001\u001d"+ - "\u0005\u001d\u0192\b\u001d\n\u001d\f\u001d\u0195\t\u001d\u0001\u001e\u0001"+ - "\u001e\u0001\u001e\u0005\u001e\u019a\b\u001e\n\u001e\f\u001e\u019d\t\u001e"+ - "\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01a2\b\u001f\n\u001f"+ - "\f\u001f\u01a5\t\u001f\u0001 \u0001 \u0001!\u0001!\u0001!\u0003!\u01ac"+ - "\b!\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ - "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01bb\b\"\n\"\f\"\u01be\t\""+ - "\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01c6\b\"\n\""+ - "\f\"\u01c9\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\""+ - "\u01d1\b\"\n\"\f\"\u01d4\t\"\u0001\"\u0001\"\u0003\"\u01d8\b\"\u0001#"+ - "\u0001#\u0003#\u01dc\b#\u0001$\u0001$\u0001$\u0003$\u01e1\b$\u0001%\u0001"+ - "%\u0001%\u0001&\u0001&\u0001&\u0001&\u0005&\u01ea\b&\n&\f&\u01ed\t&\u0001"+ - "\'\u0001\'\u0003\'\u01f1\b\'\u0001\'\u0001\'\u0003\'\u01f5\b\'\u0001("+ - "\u0001(\u0001(\u0001)\u0001)\u0001)\u0001*\u0001*\u0001*\u0001*\u0005"+ - "*\u0201\b*\n*\f*\u0204\t*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001"+ - ",\u0001,\u0003,\u020e\b,\u0001-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001"+ - ".\u0001/\u0001/\u0001/\u0005/\u021a\b/\n/\f/\u021d\t/\u00010\u00010\u0001"+ - "0\u00010\u00011\u00011\u00012\u00012\u00032\u0227\b2\u00013\u00033\u022a"+ - "\b3\u00013\u00013\u00014\u00034\u022f\b4\u00014\u00014\u00015\u00015\u0001"+ - "6\u00016\u00017\u00017\u00017\u00018\u00018\u00018\u00018\u00019\u0001"+ - "9\u00019\u0001:\u0001:\u0001:\u0001:\u0003:\u0245\b:\u0001:\u0001:\u0001"+ - ":\u0001:\u0005:\u024b\b:\n:\f:\u024e\t:\u0003:\u0250\b:\u0001;\u0001;"+ - "\u0001;\u0003;\u0255\b;\u0001;\u0001;\u0001<\u0001<\u0001<\u0001<\u0001"+ - "<\u0001=\u0001=\u0001=\u0001=\u0003=\u0262\b=\u0001>\u0003>\u0265\b>\u0001"+ - ">\u0001>\u0001>\u0001>\u0001?\u0001?\u0001?\u0003?\u026e\b?\u0001@\u0001"+ - "@\u0001@\u0001@\u0005@\u0274\b@\n@\f@\u0277\t@\u0001A\u0001A\u0001A\u0000"+ - "\u0004\u0002\n\u0012\u0014B\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+ - "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+ - "TVXZ\\^`bdfhjlnprtvxz|~\u0080\u0082\u0000\t\u0001\u0000@A\u0001\u0000"+ - "BD\u0002\u0000\u001e\u001eQQ\u0001\u0000HI\u0002\u0000##((\u0002\u0000"+ - "++..\u0002\u0000**88\u0002\u000099;?\u0001\u0000\u0016\u0018\u0294\u0000"+ - "\u0084\u0001\u0000\u0000\u0000\u0002\u0087\u0001\u0000\u0000\u0000\u0004"+ - "\u0098\u0001\u0000\u0000\u0000\u0006\u00ac\u0001\u0000\u0000\u0000\b\u00ae"+ - "\u0001\u0000\u0000\u0000\n\u00ce\u0001\u0000\u0000\u0000\f\u00e9\u0001"+ - "\u0000\u0000\u0000\u000e\u00eb\u0001\u0000\u0000\u0000\u0010\u00f4\u0001"+ - "\u0000\u0000\u0000\u0012\u00fa\u0001\u0000\u0000\u0000\u0014\u010f\u0001"+ - "\u0000\u0000\u0000\u0016\u0119\u0001\u0000\u0000\u0000\u0018\u0128\u0001"+ - "\u0000\u0000\u0000\u001a\u012a\u0001\u0000\u0000\u0000\u001c\u012c\u0001"+ - "\u0000\u0000\u0000\u001e\u012f\u0001\u0000\u0000\u0000 \u013a\u0001\u0000"+ - "\u0000\u0000\"\u013e\u0001\u0000\u0000\u0000$\u014d\u0001\u0000\u0000"+ - "\u0000&\u0151\u0001\u0000\u0000\u0000(\u0153\u0001\u0000\u0000\u0000*"+ - "\u0157\u0001\u0000\u0000\u0000,\u0159\u0001\u0000\u0000\u0000.\u0162\u0001"+ - "\u0000\u0000\u00000\u0166\u0001\u0000\u0000\u00002\u0176\u0001\u0000\u0000"+ - "\u00004\u0179\u0001\u0000\u0000\u00006\u0181\u0001\u0000\u0000\u00008"+ - "\u0189\u0001\u0000\u0000\u0000:\u018e\u0001\u0000\u0000\u0000<\u0196\u0001"+ - "\u0000\u0000\u0000>\u019e\u0001\u0000\u0000\u0000@\u01a6\u0001\u0000\u0000"+ - "\u0000B\u01ab\u0001\u0000\u0000\u0000D\u01d7\u0001\u0000\u0000\u0000F"+ - "\u01db\u0001\u0000\u0000\u0000H\u01e0\u0001\u0000\u0000\u0000J\u01e2\u0001"+ - "\u0000\u0000\u0000L\u01e5\u0001\u0000\u0000\u0000N\u01ee\u0001\u0000\u0000"+ - "\u0000P\u01f6\u0001\u0000\u0000\u0000R\u01f9\u0001\u0000\u0000\u0000T"+ - "\u01fc\u0001\u0000\u0000\u0000V\u0205\u0001\u0000\u0000\u0000X\u0209\u0001"+ - "\u0000\u0000\u0000Z\u020f\u0001\u0000\u0000\u0000\\\u0213\u0001\u0000"+ - "\u0000\u0000^\u0216\u0001\u0000\u0000\u0000`\u021e\u0001\u0000\u0000\u0000"+ - "b\u0222\u0001\u0000\u0000\u0000d\u0226\u0001\u0000\u0000\u0000f\u0229"+ - "\u0001\u0000\u0000\u0000h\u022e\u0001\u0000\u0000\u0000j\u0232\u0001\u0000"+ - "\u0000\u0000l\u0234\u0001\u0000\u0000\u0000n\u0236\u0001\u0000\u0000\u0000"+ - "p\u0239\u0001\u0000\u0000\u0000r\u023d\u0001\u0000\u0000\u0000t\u0240"+ - "\u0001\u0000\u0000\u0000v\u0254\u0001\u0000\u0000\u0000x\u0258\u0001\u0000"+ - "\u0000\u0000z\u025d\u0001\u0000\u0000\u0000|\u0264\u0001\u0000\u0000\u0000"+ - "~\u026a\u0001\u0000\u0000\u0000\u0080\u026f\u0001\u0000\u0000\u0000\u0082"+ - "\u0278\u0001\u0000\u0000\u0000\u0084\u0085\u0003\u0002\u0001\u0000\u0085"+ - "\u0086\u0005\u0000\u0000\u0001\u0086\u0001\u0001\u0000\u0000\u0000\u0087"+ - "\u0088\u0006\u0001\uffff\uffff\u0000\u0088\u0089\u0003\u0004\u0002\u0000"+ - "\u0089\u008f\u0001\u0000\u0000\u0000\u008a\u008b\n\u0001\u0000\u0000\u008b"+ - "\u008c\u0005\u001d\u0000\u0000\u008c\u008e\u0003\u0006\u0003\u0000\u008d"+ - "\u008a\u0001\u0000\u0000\u0000\u008e\u0091\u0001\u0000\u0000\u0000\u008f"+ - "\u008d\u0001\u0000\u0000\u0000\u008f\u0090\u0001\u0000\u0000\u0000\u0090"+ - "\u0003\u0001\u0000\u0000\u0000\u0091\u008f\u0001\u0000\u0000\u0000\u0092"+ - "\u0099\u0003n7\u0000\u0093\u0099\u0003\"\u0011\u0000\u0094\u0099\u0003"+ - "\u001c\u000e\u0000\u0095\u0099\u0003r9\u0000\u0096\u0097\u0004\u0002\u0001"+ - "\u0000\u0097\u0099\u00030\u0018\u0000\u0098\u0092\u0001\u0000\u0000\u0000"+ - "\u0098\u0093\u0001\u0000\u0000\u0000\u0098\u0094\u0001\u0000\u0000\u0000"+ - "\u0098\u0095\u0001\u0000\u0000\u0000\u0098\u0096\u0001\u0000\u0000\u0000"+ - "\u0099\u0005\u0001\u0000\u0000\u0000\u009a\u00ad\u00032\u0019\u0000\u009b"+ - "\u00ad\u0003\b\u0004\u0000\u009c\u00ad\u0003P(\u0000\u009d\u00ad\u0003"+ - "J%\u0000\u009e\u00ad\u00034\u001a\u0000\u009f\u00ad\u0003L&\u0000\u00a0"+ - "\u00ad\u0003R)\u0000\u00a1\u00ad\u0003T*\u0000\u00a2\u00ad\u0003X,\u0000"+ - "\u00a3\u00ad\u0003Z-\u0000\u00a4\u00ad\u0003t:\u0000\u00a5\u00ad\u0003"+ - "\\.\u0000\u00a6\u00a7\u0004\u0003\u0002\u0000\u00a7\u00ad\u0003z=\u0000"+ - "\u00a8\u00a9\u0004\u0003\u0003\u0000\u00a9\u00ad\u0003x<\u0000\u00aa\u00ab"+ - "\u0004\u0003\u0004\u0000\u00ab\u00ad\u0003|>\u0000\u00ac\u009a\u0001\u0000"+ - "\u0000\u0000\u00ac\u009b\u0001\u0000\u0000\u0000\u00ac\u009c\u0001\u0000"+ - "\u0000\u0000\u00ac\u009d\u0001\u0000\u0000\u0000\u00ac\u009e\u0001\u0000"+ - "\u0000\u0000\u00ac\u009f\u0001\u0000\u0000\u0000\u00ac\u00a0\u0001\u0000"+ - "\u0000\u0000\u00ac\u00a1\u0001\u0000\u0000\u0000\u00ac\u00a2\u0001\u0000"+ - "\u0000\u0000\u00ac\u00a3\u0001\u0000\u0000\u0000\u00ac\u00a4\u0001\u0000"+ - "\u0000\u0000\u00ac\u00a5\u0001\u0000\u0000\u0000\u00ac\u00a6\u0001\u0000"+ - "\u0000\u0000\u00ac\u00a8\u0001\u0000\u0000\u0000\u00ac\u00aa\u0001\u0000"+ - "\u0000\u0000\u00ad\u0007\u0001\u0000\u0000\u0000\u00ae\u00af\u0005\u0010"+ - "\u0000\u0000\u00af\u00b0\u0003\n\u0005\u0000\u00b0\t\u0001\u0000\u0000"+ - "\u0000\u00b1\u00b2\u0006\u0005\uffff\uffff\u0000\u00b2\u00b3\u00051\u0000"+ - "\u0000\u00b3\u00cf\u0003\n\u0005\b\u00b4\u00cf\u0003\u0010\b\u0000\u00b5"+ - "\u00cf\u0003\f\u0006\u0000\u00b6\u00b8\u0003\u0010\b\u0000\u00b7\u00b9"+ - "\u00051\u0000\u0000\u00b8\u00b7\u0001\u0000\u0000\u0000\u00b8\u00b9\u0001"+ - "\u0000\u0000\u0000\u00b9\u00ba\u0001\u0000\u0000\u0000\u00ba\u00bb\u0005"+ - ",\u0000\u0000\u00bb\u00bc\u00050\u0000\u0000\u00bc\u00c1\u0003\u0010\b"+ - "\u0000\u00bd\u00be\u0005\'\u0000\u0000\u00be\u00c0\u0003\u0010\b\u0000"+ - "\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c3\u0001\u0000\u0000\u0000"+ - "\u00c1\u00bf\u0001\u0000\u0000\u0000\u00c1\u00c2\u0001\u0000\u0000\u0000"+ - "\u00c2\u00c4\u0001\u0000\u0000\u0000\u00c3\u00c1\u0001\u0000\u0000\u0000"+ - "\u00c4\u00c5\u00057\u0000\u0000\u00c5\u00cf\u0001\u0000\u0000\u0000\u00c6"+ - "\u00c7\u0003\u0010\b\u0000\u00c7\u00c9\u0005-\u0000\u0000\u00c8\u00ca"+ - "\u00051\u0000\u0000\u00c9\u00c8\u0001\u0000\u0000\u0000\u00c9\u00ca\u0001"+ - "\u0000\u0000\u0000\u00ca\u00cb\u0001\u0000\u0000\u0000\u00cb\u00cc\u0005"+ - "2\u0000\u0000\u00cc\u00cf\u0001\u0000\u0000\u0000\u00cd\u00cf\u0003\u000e"+ - "\u0007\u0000\u00ce\u00b1\u0001\u0000\u0000\u0000\u00ce\u00b4\u0001\u0000"+ - "\u0000\u0000\u00ce\u00b5\u0001\u0000\u0000\u0000\u00ce\u00b6\u0001\u0000"+ - "\u0000\u0000\u00ce\u00c6\u0001\u0000\u0000\u0000\u00ce\u00cd\u0001\u0000"+ - "\u0000\u0000\u00cf\u00d8\u0001\u0000\u0000\u0000\u00d0\u00d1\n\u0005\u0000"+ - "\u0000\u00d1\u00d2\u0005\"\u0000\u0000\u00d2\u00d7\u0003\n\u0005\u0006"+ - "\u00d3\u00d4\n\u0004\u0000\u0000\u00d4\u00d5\u00054\u0000\u0000\u00d5"+ - "\u00d7\u0003\n\u0005\u0005\u00d6\u00d0\u0001\u0000\u0000\u0000\u00d6\u00d3"+ - "\u0001\u0000\u0000\u0000\u00d7\u00da\u0001\u0000\u0000\u0000\u00d8\u00d6"+ - "\u0001\u0000\u0000\u0000\u00d8\u00d9\u0001\u0000\u0000\u0000\u00d9\u000b"+ - "\u0001\u0000\u0000\u0000\u00da\u00d8\u0001\u0000\u0000\u0000\u00db\u00dd"+ - "\u0003\u0010\b\u0000\u00dc\u00de\u00051\u0000\u0000\u00dd\u00dc\u0001"+ - "\u0000\u0000\u0000\u00dd\u00de\u0001\u0000\u0000\u0000\u00de\u00df\u0001"+ - "\u0000\u0000\u0000\u00df\u00e0\u0005/\u0000\u0000\u00e0\u00e1\u0003j5"+ - "\u0000\u00e1\u00ea\u0001\u0000\u0000\u0000\u00e2\u00e4\u0003\u0010\b\u0000"+ - "\u00e3\u00e5\u00051\u0000\u0000\u00e4\u00e3\u0001\u0000\u0000\u0000\u00e4"+ - "\u00e5\u0001\u0000\u0000\u0000\u00e5\u00e6\u0001\u0000\u0000\u0000\u00e6"+ - "\u00e7\u00056\u0000\u0000\u00e7\u00e8\u0003j5\u0000\u00e8\u00ea\u0001"+ - "\u0000\u0000\u0000\u00e9\u00db\u0001\u0000\u0000\u0000\u00e9\u00e2\u0001"+ - "\u0000\u0000\u0000\u00ea\r\u0001\u0000\u0000\u0000\u00eb\u00ec\u0003:"+ - "\u001d\u0000\u00ec\u00ed\u0005&\u0000\u0000\u00ed\u00ee\u0003D\"\u0000"+ - "\u00ee\u000f\u0001\u0000\u0000\u0000\u00ef\u00f5\u0003\u0012\t\u0000\u00f0"+ - "\u00f1\u0003\u0012\t\u0000\u00f1\u00f2\u0003l6\u0000\u00f2\u00f3\u0003"+ - "\u0012\t\u0000\u00f3\u00f5\u0001\u0000\u0000\u0000\u00f4\u00ef\u0001\u0000"+ - "\u0000\u0000\u00f4\u00f0\u0001\u0000\u0000\u0000\u00f5\u0011\u0001\u0000"+ - "\u0000\u0000\u00f6\u00f7\u0006\t\uffff\uffff\u0000\u00f7\u00fb\u0003\u0014"+ - "\n\u0000\u00f8\u00f9\u0007\u0000\u0000\u0000\u00f9\u00fb\u0003\u0012\t"+ - "\u0003\u00fa\u00f6\u0001\u0000\u0000\u0000\u00fa\u00f8\u0001\u0000\u0000"+ - "\u0000\u00fb\u0104\u0001\u0000\u0000\u0000\u00fc\u00fd\n\u0002\u0000\u0000"+ - "\u00fd\u00fe\u0007\u0001\u0000\u0000\u00fe\u0103\u0003\u0012\t\u0003\u00ff"+ - "\u0100\n\u0001\u0000\u0000\u0100\u0101\u0007\u0000\u0000\u0000\u0101\u0103"+ - "\u0003\u0012\t\u0002\u0102\u00fc\u0001\u0000\u0000\u0000\u0102\u00ff\u0001"+ - "\u0000\u0000\u0000\u0103\u0106\u0001\u0000\u0000\u0000\u0104\u0102\u0001"+ - "\u0000\u0000\u0000\u0104\u0105\u0001\u0000\u0000\u0000\u0105\u0013\u0001"+ - "\u0000\u0000\u0000\u0106\u0104\u0001\u0000\u0000\u0000\u0107\u0108\u0006"+ - "\n\uffff\uffff\u0000\u0108\u0110\u0003D\"\u0000\u0109\u0110\u0003:\u001d"+ - "\u0000\u010a\u0110\u0003\u0016\u000b\u0000\u010b\u010c\u00050\u0000\u0000"+ - "\u010c\u010d\u0003\n\u0005\u0000\u010d\u010e\u00057\u0000\u0000\u010e"+ - "\u0110\u0001\u0000\u0000\u0000\u010f\u0107\u0001\u0000\u0000\u0000\u010f"+ - "\u0109\u0001\u0000\u0000\u0000\u010f\u010a\u0001\u0000\u0000\u0000\u010f"+ - "\u010b\u0001\u0000\u0000\u0000\u0110\u0116\u0001\u0000\u0000\u0000\u0111"+ - "\u0112\n\u0001\u0000\u0000\u0112\u0113\u0005%\u0000\u0000\u0113\u0115"+ - "\u0003\u001a\r\u0000\u0114\u0111\u0001\u0000\u0000\u0000\u0115\u0118\u0001"+ - "\u0000\u0000\u0000\u0116\u0114\u0001\u0000\u0000\u0000\u0116\u0117\u0001"+ - "\u0000\u0000\u0000\u0117\u0015\u0001\u0000\u0000\u0000\u0118\u0116\u0001"+ - "\u0000\u0000\u0000\u0119\u011a\u0003\u0018\f\u0000\u011a\u0124\u00050"+ - "\u0000\u0000\u011b\u0125\u0005B\u0000\u0000\u011c\u0121\u0003\n\u0005"+ - "\u0000\u011d\u011e\u0005\'\u0000\u0000\u011e\u0120\u0003\n\u0005\u0000"+ - "\u011f\u011d\u0001\u0000\u0000\u0000\u0120\u0123\u0001\u0000\u0000\u0000"+ - "\u0121\u011f\u0001\u0000\u0000\u0000\u0121\u0122\u0001\u0000\u0000\u0000"+ - "\u0122\u0125\u0001\u0000\u0000\u0000\u0123\u0121\u0001\u0000\u0000\u0000"+ - "\u0124\u011b\u0001\u0000\u0000\u0000\u0124\u011c\u0001\u0000\u0000\u0000"+ - "\u0124\u0125\u0001\u0000\u0000\u0000\u0125\u0126\u0001\u0000\u0000\u0000"+ - "\u0126\u0127\u00057\u0000\u0000\u0127\u0017\u0001\u0000\u0000\u0000\u0128"+ - "\u0129\u0003H$\u0000\u0129\u0019\u0001\u0000\u0000\u0000\u012a\u012b\u0003"+ - "@ \u0000\u012b\u001b\u0001\u0000\u0000\u0000\u012c\u012d\u0005\f\u0000"+ - "\u0000\u012d\u012e\u0003\u001e\u000f\u0000\u012e\u001d\u0001\u0000\u0000"+ - "\u0000\u012f\u0134\u0003 \u0010\u0000\u0130\u0131\u0005\'\u0000\u0000"+ - "\u0131\u0133\u0003 \u0010\u0000\u0132\u0130\u0001\u0000\u0000\u0000\u0133"+ - "\u0136\u0001\u0000\u0000\u0000\u0134\u0132\u0001\u0000\u0000\u0000\u0134"+ - "\u0135\u0001\u0000\u0000\u0000\u0135\u001f\u0001\u0000\u0000\u0000\u0136"+ - "\u0134\u0001\u0000\u0000\u0000\u0137\u0138\u0003:\u001d\u0000\u0138\u0139"+ - "\u0005$\u0000\u0000\u0139\u013b\u0001\u0000\u0000\u0000\u013a\u0137\u0001"+ - "\u0000\u0000\u0000\u013a\u013b\u0001\u0000\u0000\u0000\u013b\u013c\u0001"+ - "\u0000\u0000\u0000\u013c\u013d\u0003\n\u0005\u0000\u013d!\u0001\u0000"+ - "\u0000\u0000\u013e\u013f\u0005\u0006\u0000\u0000\u013f\u0144\u0003$\u0012"+ - "\u0000\u0140\u0141\u0005\'\u0000\u0000\u0141\u0143\u0003$\u0012\u0000"+ - "\u0142\u0140\u0001\u0000\u0000\u0000\u0143\u0146\u0001\u0000\u0000\u0000"+ - "\u0144\u0142\u0001\u0000\u0000\u0000\u0144\u0145\u0001\u0000\u0000\u0000"+ - "\u0145\u0148\u0001\u0000\u0000\u0000\u0146\u0144\u0001\u0000\u0000\u0000"+ - "\u0147\u0149\u0003*\u0015\u0000\u0148\u0147\u0001\u0000\u0000\u0000\u0148"+ - "\u0149\u0001\u0000\u0000\u0000\u0149#\u0001\u0000\u0000\u0000\u014a\u014b"+ - "\u0003&\u0013\u0000\u014b\u014c\u0005&\u0000\u0000\u014c\u014e\u0001\u0000"+ - "\u0000\u0000\u014d\u014a\u0001\u0000\u0000\u0000\u014d\u014e\u0001\u0000"+ - "\u0000\u0000\u014e\u014f\u0001\u0000\u0000\u0000\u014f\u0150\u0003(\u0014"+ - "\u0000\u0150%\u0001\u0000\u0000\u0000\u0151\u0152\u0005Q\u0000\u0000\u0152"+ - "\'\u0001\u0000\u0000\u0000\u0153\u0154\u0007\u0002\u0000\u0000\u0154)"+ - "\u0001\u0000\u0000\u0000\u0155\u0158\u0003,\u0016\u0000\u0156\u0158\u0003"+ - ".\u0017\u0000\u0157\u0155\u0001\u0000\u0000\u0000\u0157\u0156\u0001\u0000"+ - "\u0000\u0000\u0158+\u0001\u0000\u0000\u0000\u0159\u015a\u0005P\u0000\u0000"+ - "\u015a\u015f\u0005Q\u0000\u0000\u015b\u015c\u0005\'\u0000\u0000\u015c"+ - "\u015e\u0005Q\u0000\u0000\u015d\u015b\u0001\u0000\u0000\u0000\u015e\u0161"+ - "\u0001\u0000\u0000\u0000\u015f\u015d\u0001\u0000\u0000\u0000\u015f\u0160"+ - "\u0001\u0000\u0000\u0000\u0160-\u0001\u0000\u0000\u0000\u0161\u015f\u0001"+ - "\u0000\u0000\u0000\u0162\u0163\u0005F\u0000\u0000\u0163\u0164\u0003,\u0016"+ - "\u0000\u0164\u0165\u0005G\u0000\u0000\u0165/\u0001\u0000\u0000\u0000\u0166"+ - "\u0167\u0005\u0013\u0000\u0000\u0167\u016c\u0003$\u0012\u0000\u0168\u0169"+ - "\u0005\'\u0000\u0000\u0169\u016b\u0003$\u0012\u0000\u016a\u0168\u0001"+ - "\u0000\u0000\u0000\u016b\u016e\u0001\u0000\u0000\u0000\u016c\u016a\u0001"+ - "\u0000\u0000\u0000\u016c\u016d\u0001\u0000\u0000\u0000\u016d\u0170\u0001"+ - "\u0000\u0000\u0000\u016e\u016c\u0001\u0000\u0000\u0000\u016f\u0171\u0003"+ - "6\u001b\u0000\u0170\u016f\u0001\u0000\u0000\u0000\u0170\u0171\u0001\u0000"+ - "\u0000\u0000\u0171\u0174\u0001\u0000\u0000\u0000\u0172\u0173\u0005!\u0000"+ - "\u0000\u0173\u0175\u0003\u001e\u000f\u0000\u0174\u0172\u0001\u0000\u0000"+ - "\u0000\u0174\u0175\u0001\u0000\u0000\u0000\u01751\u0001\u0000\u0000\u0000"+ - "\u0176\u0177\u0005\u0004\u0000\u0000\u0177\u0178\u0003\u001e\u000f\u0000"+ - "\u01783\u0001\u0000\u0000\u0000\u0179\u017b\u0005\u000f\u0000\u0000\u017a"+ - "\u017c\u00036\u001b\u0000\u017b\u017a\u0001\u0000\u0000\u0000\u017b\u017c"+ - "\u0001\u0000\u0000\u0000\u017c\u017f\u0001\u0000\u0000\u0000\u017d\u017e"+ - "\u0005!\u0000\u0000\u017e\u0180\u0003\u001e\u000f\u0000\u017f\u017d\u0001"+ - "\u0000\u0000\u0000\u017f\u0180\u0001\u0000\u0000\u0000\u01805\u0001\u0000"+ - "\u0000\u0000\u0181\u0186\u00038\u001c\u0000\u0182\u0183\u0005\'\u0000"+ - "\u0000\u0183\u0185\u00038\u001c\u0000\u0184\u0182\u0001\u0000\u0000\u0000"+ - "\u0185\u0188\u0001\u0000\u0000\u0000\u0186\u0184\u0001\u0000\u0000\u0000"+ - "\u0186\u0187\u0001\u0000\u0000\u0000\u01877\u0001\u0000\u0000\u0000\u0188"+ - "\u0186\u0001\u0000\u0000\u0000\u0189\u018c\u0003 \u0010\u0000\u018a\u018b"+ - "\u0005\u0010\u0000\u0000\u018b\u018d\u0003\n\u0005\u0000\u018c\u018a\u0001"+ - "\u0000\u0000\u0000\u018c\u018d\u0001\u0000\u0000\u0000\u018d9\u0001\u0000"+ - "\u0000\u0000\u018e\u0193\u0003H$\u0000\u018f\u0190\u0005)\u0000\u0000"+ - "\u0190\u0192\u0003H$\u0000\u0191\u018f\u0001\u0000\u0000\u0000\u0192\u0195"+ - "\u0001\u0000\u0000\u0000\u0193\u0191\u0001\u0000\u0000\u0000\u0193\u0194"+ - "\u0001\u0000\u0000\u0000\u0194;\u0001\u0000\u0000\u0000\u0195\u0193\u0001"+ - "\u0000\u0000\u0000\u0196\u019b\u0003B!\u0000\u0197\u0198\u0005)\u0000"+ - "\u0000\u0198\u019a\u0003B!\u0000\u0199\u0197\u0001\u0000\u0000\u0000\u019a"+ - "\u019d\u0001\u0000\u0000\u0000\u019b\u0199\u0001\u0000\u0000\u0000\u019b"+ - "\u019c\u0001\u0000\u0000\u0000\u019c=\u0001\u0000\u0000\u0000\u019d\u019b"+ - "\u0001\u0000\u0000\u0000\u019e\u01a3\u0003<\u001e\u0000\u019f\u01a0\u0005"+ - "\'\u0000\u0000\u01a0\u01a2\u0003<\u001e\u0000\u01a1\u019f\u0001\u0000"+ - "\u0000\u0000\u01a2\u01a5\u0001\u0000\u0000\u0000\u01a3\u01a1\u0001\u0000"+ - "\u0000\u0000\u01a3\u01a4\u0001\u0000\u0000\u0000\u01a4?\u0001\u0000\u0000"+ - "\u0000\u01a5\u01a3\u0001\u0000\u0000\u0000\u01a6\u01a7\u0007\u0003\u0000"+ - "\u0000\u01a7A\u0001\u0000\u0000\u0000\u01a8\u01ac\u0005U\u0000\u0000\u01a9"+ - "\u01aa\u0004!\n\u0000\u01aa\u01ac\u0003F#\u0000\u01ab\u01a8\u0001\u0000"+ - "\u0000\u0000\u01ab\u01a9\u0001\u0000\u0000\u0000\u01acC\u0001\u0000\u0000"+ - "\u0000\u01ad\u01d8\u00052\u0000\u0000\u01ae\u01af\u0003h4\u0000\u01af"+ - "\u01b0\u0005H\u0000\u0000\u01b0\u01d8\u0001\u0000\u0000\u0000\u01b1\u01d8"+ - "\u0003f3\u0000\u01b2\u01d8\u0003h4\u0000\u01b3\u01d8\u0003b1\u0000\u01b4"+ - "\u01d8\u0003F#\u0000\u01b5\u01d8\u0003j5\u0000\u01b6\u01b7\u0005F\u0000"+ - "\u0000\u01b7\u01bc\u0003d2\u0000\u01b8\u01b9\u0005\'\u0000\u0000\u01b9"+ - "\u01bb\u0003d2\u0000\u01ba\u01b8\u0001\u0000\u0000\u0000\u01bb\u01be\u0001"+ - "\u0000\u0000\u0000\u01bc\u01ba\u0001\u0000\u0000\u0000\u01bc\u01bd\u0001"+ - "\u0000\u0000\u0000\u01bd\u01bf\u0001\u0000\u0000\u0000\u01be\u01bc\u0001"+ - "\u0000\u0000\u0000\u01bf\u01c0\u0005G\u0000\u0000\u01c0\u01d8\u0001\u0000"+ - "\u0000\u0000\u01c1\u01c2\u0005F\u0000\u0000\u01c2\u01c7\u0003b1\u0000"+ - "\u01c3\u01c4\u0005\'\u0000\u0000\u01c4\u01c6\u0003b1\u0000\u01c5\u01c3"+ - "\u0001\u0000\u0000\u0000\u01c6\u01c9\u0001\u0000\u0000\u0000\u01c7\u01c5"+ - "\u0001\u0000\u0000\u0000\u01c7\u01c8\u0001\u0000\u0000\u0000\u01c8\u01ca"+ - "\u0001\u0000\u0000\u0000\u01c9\u01c7\u0001\u0000\u0000\u0000\u01ca\u01cb"+ - "\u0005G\u0000\u0000\u01cb\u01d8\u0001\u0000\u0000\u0000\u01cc\u01cd\u0005"+ - "F\u0000\u0000\u01cd\u01d2\u0003j5\u0000\u01ce\u01cf\u0005\'\u0000\u0000"+ - "\u01cf\u01d1\u0003j5\u0000\u01d0\u01ce\u0001\u0000\u0000\u0000\u01d1\u01d4"+ - "\u0001\u0000\u0000\u0000\u01d2\u01d0\u0001\u0000\u0000\u0000\u01d2\u01d3"+ - "\u0001\u0000\u0000\u0000\u01d3\u01d5\u0001\u0000\u0000\u0000\u01d4\u01d2"+ - "\u0001\u0000\u0000\u0000\u01d5\u01d6\u0005G\u0000\u0000\u01d6\u01d8\u0001"+ - "\u0000\u0000\u0000\u01d7\u01ad\u0001\u0000\u0000\u0000\u01d7\u01ae\u0001"+ - "\u0000\u0000\u0000\u01d7\u01b1\u0001\u0000\u0000\u0000\u01d7\u01b2\u0001"+ - "\u0000\u0000\u0000\u01d7\u01b3\u0001\u0000\u0000\u0000\u01d7\u01b4\u0001"+ - "\u0000\u0000\u0000\u01d7\u01b5\u0001\u0000\u0000\u0000\u01d7\u01b6\u0001"+ - "\u0000\u0000\u0000\u01d7\u01c1\u0001\u0000\u0000\u0000\u01d7\u01cc\u0001"+ - "\u0000\u0000\u0000\u01d8E\u0001\u0000\u0000\u0000\u01d9\u01dc\u00055\u0000"+ - "\u0000\u01da\u01dc\u0005E\u0000\u0000\u01db\u01d9\u0001\u0000\u0000\u0000"+ - "\u01db\u01da\u0001\u0000\u0000\u0000\u01dcG\u0001\u0000\u0000\u0000\u01dd"+ - "\u01e1\u0003@ \u0000\u01de\u01df\u0004$\u000b\u0000\u01df\u01e1\u0003"+ - "F#\u0000\u01e0\u01dd\u0001\u0000\u0000\u0000\u01e0\u01de\u0001\u0000\u0000"+ - "\u0000\u01e1I\u0001\u0000\u0000\u0000\u01e2\u01e3\u0005\t\u0000\u0000"+ - "\u01e3\u01e4\u0005\u001f\u0000\u0000\u01e4K\u0001\u0000\u0000\u0000\u01e5"+ - "\u01e6\u0005\u000e\u0000\u0000\u01e6\u01eb\u0003N\'\u0000\u01e7\u01e8"+ - "\u0005\'\u0000\u0000\u01e8\u01ea\u0003N\'\u0000\u01e9\u01e7\u0001\u0000"+ - "\u0000\u0000\u01ea\u01ed\u0001\u0000\u0000\u0000\u01eb\u01e9\u0001\u0000"+ - "\u0000\u0000\u01eb\u01ec\u0001\u0000\u0000\u0000\u01ecM\u0001\u0000\u0000"+ - "\u0000\u01ed\u01eb\u0001\u0000\u0000\u0000\u01ee\u01f0\u0003\n\u0005\u0000"+ - "\u01ef\u01f1\u0007\u0004\u0000\u0000\u01f0\u01ef\u0001\u0000\u0000\u0000"+ - "\u01f0\u01f1\u0001\u0000\u0000\u0000\u01f1\u01f4\u0001\u0000\u0000\u0000"+ - "\u01f2\u01f3\u00053\u0000\u0000\u01f3\u01f5\u0007\u0005\u0000\u0000\u01f4"+ - "\u01f2\u0001\u0000\u0000\u0000\u01f4\u01f5\u0001\u0000\u0000\u0000\u01f5"+ - "O\u0001\u0000\u0000\u0000\u01f6\u01f7\u0005\b\u0000\u0000\u01f7\u01f8"+ - "\u0003>\u001f\u0000\u01f8Q\u0001\u0000\u0000\u0000\u01f9\u01fa\u0005\u0002"+ - "\u0000\u0000\u01fa\u01fb\u0003>\u001f\u0000\u01fbS\u0001\u0000\u0000\u0000"+ - "\u01fc\u01fd\u0005\u000b\u0000\u0000\u01fd\u0202\u0003V+\u0000\u01fe\u01ff"+ - "\u0005\'\u0000\u0000\u01ff\u0201\u0003V+\u0000\u0200\u01fe\u0001\u0000"+ - "\u0000\u0000\u0201\u0204\u0001\u0000\u0000\u0000\u0202\u0200\u0001\u0000"+ - "\u0000\u0000\u0202\u0203\u0001\u0000\u0000\u0000\u0203U\u0001\u0000\u0000"+ - "\u0000\u0204\u0202\u0001\u0000\u0000\u0000\u0205\u0206\u0003<\u001e\u0000"+ - "\u0206\u0207\u0005Y\u0000\u0000\u0207\u0208\u0003<\u001e\u0000\u0208W"+ - "\u0001\u0000\u0000\u0000\u0209\u020a\u0005\u0001\u0000\u0000\u020a\u020b"+ - "\u0003\u0014\n\u0000\u020b\u020d\u0003j5\u0000\u020c\u020e\u0003^/\u0000"+ - "\u020d\u020c\u0001\u0000\u0000\u0000\u020d\u020e\u0001\u0000\u0000\u0000"+ - "\u020eY\u0001\u0000\u0000\u0000\u020f\u0210\u0005\u0007\u0000\u0000\u0210"+ - "\u0211\u0003\u0014\n\u0000\u0211\u0212\u0003j5\u0000\u0212[\u0001\u0000"+ - "\u0000\u0000\u0213\u0214\u0005\n\u0000\u0000\u0214\u0215\u0003:\u001d"+ - "\u0000\u0215]\u0001\u0000\u0000\u0000\u0216\u021b\u0003`0\u0000\u0217"+ - "\u0218\u0005\'\u0000\u0000\u0218\u021a\u0003`0\u0000\u0219\u0217\u0001"+ - "\u0000\u0000\u0000\u021a\u021d\u0001\u0000\u0000\u0000\u021b\u0219\u0001"+ - "\u0000\u0000\u0000\u021b\u021c\u0001\u0000\u0000\u0000\u021c_\u0001\u0000"+ - "\u0000\u0000\u021d\u021b\u0001\u0000\u0000\u0000\u021e\u021f\u0003@ \u0000"+ - "\u021f\u0220\u0005$\u0000\u0000\u0220\u0221\u0003D\"\u0000\u0221a\u0001"+ - "\u0000\u0000\u0000\u0222\u0223\u0007\u0006\u0000\u0000\u0223c\u0001\u0000"+ - "\u0000\u0000\u0224\u0227\u0003f3\u0000\u0225\u0227\u0003h4\u0000\u0226"+ - "\u0224\u0001\u0000\u0000\u0000\u0226\u0225\u0001\u0000\u0000\u0000\u0227"+ - "e\u0001\u0000\u0000\u0000\u0228\u022a\u0007\u0000\u0000\u0000\u0229\u0228"+ - "\u0001\u0000\u0000\u0000\u0229\u022a\u0001\u0000\u0000\u0000\u022a\u022b"+ - "\u0001\u0000\u0000\u0000\u022b\u022c\u0005 \u0000\u0000\u022cg\u0001\u0000"+ - "\u0000\u0000\u022d\u022f\u0007\u0000\u0000\u0000\u022e\u022d\u0001\u0000"+ - "\u0000\u0000\u022e\u022f\u0001\u0000\u0000\u0000\u022f\u0230\u0001\u0000"+ - "\u0000\u0000\u0230\u0231\u0005\u001f\u0000\u0000\u0231i\u0001\u0000\u0000"+ - "\u0000\u0232\u0233\u0005\u001e\u0000\u0000\u0233k\u0001\u0000\u0000\u0000"+ - "\u0234\u0235\u0007\u0007\u0000\u0000\u0235m\u0001\u0000\u0000\u0000\u0236"+ - "\u0237\u0005\u0005\u0000\u0000\u0237\u0238\u0003p8\u0000\u0238o\u0001"+ - "\u0000\u0000\u0000\u0239\u023a\u0005F\u0000\u0000\u023a\u023b\u0003\u0002"+ - "\u0001\u0000\u023b\u023c\u0005G\u0000\u0000\u023cq\u0001\u0000\u0000\u0000"+ - "\u023d\u023e\u0005\r\u0000\u0000\u023e\u023f\u0005i\u0000\u0000\u023f"+ - "s\u0001\u0000\u0000\u0000\u0240\u0241\u0005\u0003\u0000\u0000\u0241\u0244"+ - "\u0005_\u0000\u0000\u0242\u0243\u0005]\u0000\u0000\u0243\u0245\u0003<"+ - "\u001e\u0000\u0244\u0242\u0001\u0000\u0000\u0000\u0244\u0245\u0001\u0000"+ - "\u0000\u0000\u0245\u024f\u0001\u0000\u0000\u0000\u0246\u0247\u0005^\u0000"+ - "\u0000\u0247\u024c\u0003v;\u0000\u0248\u0249\u0005\'\u0000\u0000\u0249"+ - "\u024b\u0003v;\u0000\u024a\u0248\u0001\u0000\u0000\u0000\u024b\u024e\u0001"+ - "\u0000\u0000\u0000\u024c\u024a\u0001\u0000\u0000\u0000\u024c\u024d\u0001"+ - "\u0000\u0000\u0000\u024d\u0250\u0001\u0000\u0000\u0000\u024e\u024c\u0001"+ - "\u0000\u0000\u0000\u024f\u0246\u0001\u0000\u0000\u0000\u024f\u0250\u0001"+ - "\u0000\u0000\u0000\u0250u\u0001\u0000\u0000\u0000\u0251\u0252\u0003<\u001e"+ - "\u0000\u0252\u0253\u0005$\u0000\u0000\u0253\u0255\u0001\u0000\u0000\u0000"+ - "\u0254\u0251\u0001\u0000\u0000\u0000\u0254\u0255\u0001\u0000\u0000\u0000"+ - "\u0255\u0256\u0001\u0000\u0000\u0000\u0256\u0257\u0003<\u001e\u0000\u0257"+ - "w\u0001\u0000\u0000\u0000\u0258\u0259\u0005\u0012\u0000\u0000\u0259\u025a"+ - "\u0003$\u0012\u0000\u025a\u025b\u0005]\u0000\u0000\u025b\u025c\u0003>"+ - "\u001f\u0000\u025cy\u0001\u0000\u0000\u0000\u025d\u025e\u0005\u0011\u0000"+ - "\u0000\u025e\u0261\u00036\u001b\u0000\u025f\u0260\u0005!\u0000\u0000\u0260"+ - "\u0262\u0003\u001e\u000f\u0000\u0261\u025f\u0001\u0000\u0000\u0000\u0261"+ - "\u0262\u0001\u0000\u0000\u0000\u0262{\u0001\u0000\u0000\u0000\u0263\u0265"+ - "\u0007\b\u0000\u0000\u0264\u0263\u0001\u0000\u0000\u0000\u0264\u0265\u0001"+ - "\u0000\u0000\u0000\u0265\u0266\u0001\u0000\u0000\u0000\u0266\u0267\u0005"+ - "\u0014\u0000\u0000\u0267\u0268\u0003~?\u0000\u0268\u0269\u0003\u0080@"+ - "\u0000\u0269}\u0001\u0000\u0000\u0000\u026a\u026d\u0003@ \u0000\u026b"+ - "\u026c\u0005Y\u0000\u0000\u026c\u026e\u0003@ \u0000\u026d\u026b\u0001"+ - "\u0000\u0000\u0000\u026d\u026e\u0001\u0000\u0000\u0000\u026e\u007f\u0001"+ - "\u0000\u0000\u0000\u026f\u0270\u0005]\u0000\u0000\u0270\u0275\u0003\u0082"+ - "A\u0000\u0271\u0272\u0005\'\u0000\u0000\u0272\u0274\u0003\u0082A\u0000"+ - "\u0273\u0271\u0001\u0000\u0000\u0000\u0274\u0277\u0001\u0000\u0000\u0000"+ - "\u0275\u0273\u0001\u0000\u0000\u0000\u0275\u0276\u0001\u0000\u0000\u0000"+ - "\u0276\u0081\u0001\u0000\u0000\u0000\u0277\u0275\u0001\u0000\u0000\u0000"+ - "\u0278\u0279\u0003\u0010\b\u0000\u0279\u0083\u0001\u0000\u0000\u0000="+ - "\u008f\u0098\u00ac\u00b8\u00c1\u00c9\u00ce\u00d6\u00d8\u00dd\u00e4\u00e9"+ - "\u00f4\u00fa\u0102\u0104\u010f\u0116\u0121\u0124\u0134\u013a\u0144\u0148"+ - "\u014d\u0157\u015f\u016c\u0170\u0174\u017b\u017f\u0186\u018c\u0193\u019b"+ - "\u01a3\u01ab\u01bc\u01c7\u01d2\u01d7\u01db\u01e0\u01eb\u01f0\u01f4\u0202"+ - "\u020d\u021b\u0226\u0229\u022e\u0244\u024c\u024f\u0254\u0261\u0264\u026d"+ - "\u0275"; + "\u00ea\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u00ef\b"+ + "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001"+ + "\b\u0001\b\u0003\b\u00f9\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00ff"+ + "\b\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u0107\b\t"+ + "\n\t\f\t\u010a\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ + "\n\u0001\n\u0003\n\u0114\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0119\b\n"+ + "\n\n\f\n\u011c\t\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001"+ + "\u000b\u0001\u000b\u0005\u000b\u0124\b\u000b\n\u000b\f\u000b\u0127\t\u000b"+ + "\u0003\u000b\u0129\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001"+ + "\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f"+ + "\u0001\u000f\u0005\u000f\u0137\b\u000f\n\u000f\f\u000f\u013a\t\u000f\u0001"+ + "\u0010\u0001\u0010\u0001\u0010\u0003\u0010\u013f\b\u0010\u0001\u0010\u0001"+ + "\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0005\u0011\u0147"+ + "\b\u0011\n\u0011\f\u0011\u014a\t\u0011\u0001\u0011\u0003\u0011\u014d\b"+ + "\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0003\u0012\u0152\b\u0012\u0001"+ + "\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001"+ + "\u0015\u0001\u0015\u0003\u0015\u015c\b\u0015\u0001\u0016\u0001\u0016\u0001"+ + "\u0016\u0001\u0016\u0005\u0016\u0162\b\u0016\n\u0016\f\u0016\u0165\t\u0016"+ + "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018"+ + "\u0001\u0018\u0001\u0018\u0005\u0018\u016f\b\u0018\n\u0018\f\u0018\u0172"+ + "\t\u0018\u0001\u0018\u0003\u0018\u0175\b\u0018\u0001\u0018\u0001\u0018"+ + "\u0003\u0018\u0179\b\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a"+ + "\u0001\u001a\u0003\u001a\u0180\b\u001a\u0001\u001a\u0001\u001a\u0003\u001a"+ + "\u0184\b\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0005\u001b\u0189\b"+ + "\u001b\n\u001b\f\u001b\u018c\t\u001b\u0001\u001c\u0001\u001c\u0001\u001c"+ + "\u0003\u001c\u0191\b\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d"+ + "\u0196\b\u001d\n\u001d\f\u001d\u0199\t\u001d\u0001\u001e\u0001\u001e\u0001"+ + "\u001e\u0005\u001e\u019e\b\u001e\n\u001e\f\u001e\u01a1\t\u001e\u0001\u001f"+ + "\u0001\u001f\u0001\u001f\u0005\u001f\u01a6\b\u001f\n\u001f\f\u001f\u01a9"+ + "\t\u001f\u0001 \u0001 \u0001!\u0001!\u0001!\u0003!\u01b0\b!\u0001\"\u0001"+ + "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ + "\"\u0001\"\u0001\"\u0005\"\u01bf\b\"\n\"\f\"\u01c2\t\"\u0001\"\u0001\""+ + "\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01ca\b\"\n\"\f\"\u01cd\t\"\u0001"+ + "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01d5\b\"\n\"\f\"\u01d8"+ + "\t\"\u0001\"\u0001\"\u0003\"\u01dc\b\"\u0001#\u0001#\u0003#\u01e0\b#\u0001"+ + "$\u0001$\u0001$\u0003$\u01e5\b$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001"+ + "&\u0001&\u0005&\u01ee\b&\n&\f&\u01f1\t&\u0001\'\u0001\'\u0003\'\u01f5"+ + "\b\'\u0001\'\u0001\'\u0003\'\u01f9\b\'\u0001(\u0001(\u0001(\u0001)\u0001"+ + ")\u0001)\u0001*\u0001*\u0001*\u0001*\u0005*\u0205\b*\n*\f*\u0208\t*\u0001"+ + "+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001,\u0003,\u0212\b,\u0001"+ + "-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001.\u0001/\u0001/\u0001/\u0005"+ + "/\u021e\b/\n/\f/\u0221\t/\u00010\u00010\u00010\u00010\u00011\u00011\u0001"+ + "2\u00012\u00032\u022b\b2\u00013\u00033\u022e\b3\u00013\u00013\u00014\u0003"+ + "4\u0233\b4\u00014\u00014\u00015\u00015\u00016\u00016\u00017\u00017\u0001"+ + "7\u00018\u00018\u00018\u00018\u00019\u00019\u00019\u0001:\u0001:\u0001"+ + ":\u0001:\u0003:\u0249\b:\u0001:\u0001:\u0001:\u0001:\u0005:\u024f\b:\n"+ + ":\f:\u0252\t:\u0003:\u0254\b:\u0001;\u0001;\u0001;\u0003;\u0259\b;\u0001"+ + ";\u0001;\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001"+ + "=\u0003=\u0266\b=\u0001>\u0003>\u0269\b>\u0001>\u0001>\u0001>\u0001>\u0001"+ + "?\u0001?\u0001?\u0003?\u0272\b?\u0001@\u0001@\u0001@\u0001@\u0005@\u0278"+ + "\b@\n@\f@\u027b\t@\u0001A\u0001A\u0001A\u0000\u0004\u0002\n\u0012\u0014"+ + "B\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a"+ + "\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\^`bdfhjlnprtvxz|~\u0080\u0082"+ + "\u0000\t\u0001\u0000@A\u0001\u0000BD\u0002\u0000\u001e\u001eQQ\u0001\u0000"+ + "HI\u0002\u0000##((\u0002\u0000++..\u0002\u0000**88\u0002\u000099;?\u0001"+ + "\u0000\u0016\u0018\u0299\u0000\u0084\u0001\u0000\u0000\u0000\u0002\u0087"+ + "\u0001\u0000\u0000\u0000\u0004\u0098\u0001\u0000\u0000\u0000\u0006\u00ac"+ + "\u0001\u0000\u0000\u0000\b\u00ae\u0001\u0000\u0000\u0000\n\u00ce\u0001"+ + "\u0000\u0000\u0000\f\u00e9\u0001\u0000\u0000\u0000\u000e\u00eb\u0001\u0000"+ + "\u0000\u0000\u0010\u00f8\u0001\u0000\u0000\u0000\u0012\u00fe\u0001\u0000"+ + "\u0000\u0000\u0014\u0113\u0001\u0000\u0000\u0000\u0016\u011d\u0001\u0000"+ + "\u0000\u0000\u0018\u012c\u0001\u0000\u0000\u0000\u001a\u012e\u0001\u0000"+ + "\u0000\u0000\u001c\u0130\u0001\u0000\u0000\u0000\u001e\u0133\u0001\u0000"+ + "\u0000\u0000 \u013e\u0001\u0000\u0000\u0000\"\u0142\u0001\u0000\u0000"+ + "\u0000$\u0151\u0001\u0000\u0000\u0000&\u0155\u0001\u0000\u0000\u0000("+ + "\u0157\u0001\u0000\u0000\u0000*\u015b\u0001\u0000\u0000\u0000,\u015d\u0001"+ + "\u0000\u0000\u0000.\u0166\u0001\u0000\u0000\u00000\u016a\u0001\u0000\u0000"+ + "\u00002\u017a\u0001\u0000\u0000\u00004\u017d\u0001\u0000\u0000\u00006"+ + "\u0185\u0001\u0000\u0000\u00008\u018d\u0001\u0000\u0000\u0000:\u0192\u0001"+ + "\u0000\u0000\u0000<\u019a\u0001\u0000\u0000\u0000>\u01a2\u0001\u0000\u0000"+ + "\u0000@\u01aa\u0001\u0000\u0000\u0000B\u01af\u0001\u0000\u0000\u0000D"+ + "\u01db\u0001\u0000\u0000\u0000F\u01df\u0001\u0000\u0000\u0000H\u01e4\u0001"+ + "\u0000\u0000\u0000J\u01e6\u0001\u0000\u0000\u0000L\u01e9\u0001\u0000\u0000"+ + "\u0000N\u01f2\u0001\u0000\u0000\u0000P\u01fa\u0001\u0000\u0000\u0000R"+ + "\u01fd\u0001\u0000\u0000\u0000T\u0200\u0001\u0000\u0000\u0000V\u0209\u0001"+ + "\u0000\u0000\u0000X\u020d\u0001\u0000\u0000\u0000Z\u0213\u0001\u0000\u0000"+ + "\u0000\\\u0217\u0001\u0000\u0000\u0000^\u021a\u0001\u0000\u0000\u0000"+ + "`\u0222\u0001\u0000\u0000\u0000b\u0226\u0001\u0000\u0000\u0000d\u022a"+ + "\u0001\u0000\u0000\u0000f\u022d\u0001\u0000\u0000\u0000h\u0232\u0001\u0000"+ + "\u0000\u0000j\u0236\u0001\u0000\u0000\u0000l\u0238\u0001\u0000\u0000\u0000"+ + "n\u023a\u0001\u0000\u0000\u0000p\u023d\u0001\u0000\u0000\u0000r\u0241"+ + "\u0001\u0000\u0000\u0000t\u0244\u0001\u0000\u0000\u0000v\u0258\u0001\u0000"+ + "\u0000\u0000x\u025c\u0001\u0000\u0000\u0000z\u0261\u0001\u0000\u0000\u0000"+ + "|\u0268\u0001\u0000\u0000\u0000~\u026e\u0001\u0000\u0000\u0000\u0080\u0273"+ + "\u0001\u0000\u0000\u0000\u0082\u027c\u0001\u0000\u0000\u0000\u0084\u0085"+ + "\u0003\u0002\u0001\u0000\u0085\u0086\u0005\u0000\u0000\u0001\u0086\u0001"+ + "\u0001\u0000\u0000\u0000\u0087\u0088\u0006\u0001\uffff\uffff\u0000\u0088"+ + "\u0089\u0003\u0004\u0002\u0000\u0089\u008f\u0001\u0000\u0000\u0000\u008a"+ + "\u008b\n\u0001\u0000\u0000\u008b\u008c\u0005\u001d\u0000\u0000\u008c\u008e"+ + "\u0003\u0006\u0003\u0000\u008d\u008a\u0001\u0000\u0000\u0000\u008e\u0091"+ + "\u0001\u0000\u0000\u0000\u008f\u008d\u0001\u0000\u0000\u0000\u008f\u0090"+ + "\u0001\u0000\u0000\u0000\u0090\u0003\u0001\u0000\u0000\u0000\u0091\u008f"+ + "\u0001\u0000\u0000\u0000\u0092\u0099\u0003n7\u0000\u0093\u0099\u0003\""+ + "\u0011\u0000\u0094\u0099\u0003\u001c\u000e\u0000\u0095\u0099\u0003r9\u0000"+ + "\u0096\u0097\u0004\u0002\u0001\u0000\u0097\u0099\u00030\u0018\u0000\u0098"+ + "\u0092\u0001\u0000\u0000\u0000\u0098\u0093\u0001\u0000\u0000\u0000\u0098"+ + "\u0094\u0001\u0000\u0000\u0000\u0098\u0095\u0001\u0000\u0000\u0000\u0098"+ + "\u0096\u0001\u0000\u0000\u0000\u0099\u0005\u0001\u0000\u0000\u0000\u009a"+ + "\u00ad\u00032\u0019\u0000\u009b\u00ad\u0003\b\u0004\u0000\u009c\u00ad"+ + "\u0003P(\u0000\u009d\u00ad\u0003J%\u0000\u009e\u00ad\u00034\u001a\u0000"+ + "\u009f\u00ad\u0003L&\u0000\u00a0\u00ad\u0003R)\u0000\u00a1\u00ad\u0003"+ + "T*\u0000\u00a2\u00ad\u0003X,\u0000\u00a3\u00ad\u0003Z-\u0000\u00a4\u00ad"+ + "\u0003t:\u0000\u00a5\u00ad\u0003\\.\u0000\u00a6\u00a7\u0004\u0003\u0002"+ + "\u0000\u00a7\u00ad\u0003z=\u0000\u00a8\u00a9\u0004\u0003\u0003\u0000\u00a9"+ + "\u00ad\u0003x<\u0000\u00aa\u00ab\u0004\u0003\u0004\u0000\u00ab\u00ad\u0003"+ + "|>\u0000\u00ac\u009a\u0001\u0000\u0000\u0000\u00ac\u009b\u0001\u0000\u0000"+ + "\u0000\u00ac\u009c\u0001\u0000\u0000\u0000\u00ac\u009d\u0001\u0000\u0000"+ + "\u0000\u00ac\u009e\u0001\u0000\u0000\u0000\u00ac\u009f\u0001\u0000\u0000"+ + "\u0000\u00ac\u00a0\u0001\u0000\u0000\u0000\u00ac\u00a1\u0001\u0000\u0000"+ + "\u0000\u00ac\u00a2\u0001\u0000\u0000\u0000\u00ac\u00a3\u0001\u0000\u0000"+ + "\u0000\u00ac\u00a4\u0001\u0000\u0000\u0000\u00ac\u00a5\u0001\u0000\u0000"+ + "\u0000\u00ac\u00a6\u0001\u0000\u0000\u0000\u00ac\u00a8\u0001\u0000\u0000"+ + "\u0000\u00ac\u00aa\u0001\u0000\u0000\u0000\u00ad\u0007\u0001\u0000\u0000"+ + "\u0000\u00ae\u00af\u0005\u0010\u0000\u0000\u00af\u00b0\u0003\n\u0005\u0000"+ + "\u00b0\t\u0001\u0000\u0000\u0000\u00b1\u00b2\u0006\u0005\uffff\uffff\u0000"+ + "\u00b2\u00b3\u00051\u0000\u0000\u00b3\u00cf\u0003\n\u0005\b\u00b4\u00cf"+ + "\u0003\u0010\b\u0000\u00b5\u00cf\u0003\f\u0006\u0000\u00b6\u00b8\u0003"+ + "\u0010\b\u0000\u00b7\u00b9\u00051\u0000\u0000\u00b8\u00b7\u0001\u0000"+ + "\u0000\u0000\u00b8\u00b9\u0001\u0000\u0000\u0000\u00b9\u00ba\u0001\u0000"+ + "\u0000\u0000\u00ba\u00bb\u0005,\u0000\u0000\u00bb\u00bc\u00050\u0000\u0000"+ + "\u00bc\u00c1\u0003\u0010\b\u0000\u00bd\u00be\u0005\'\u0000\u0000\u00be"+ + "\u00c0\u0003\u0010\b\u0000\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c3"+ + "\u0001\u0000\u0000\u0000\u00c1\u00bf\u0001\u0000\u0000\u0000\u00c1\u00c2"+ + "\u0001\u0000\u0000\u0000\u00c2\u00c4\u0001\u0000\u0000\u0000\u00c3\u00c1"+ + "\u0001\u0000\u0000\u0000\u00c4\u00c5\u00057\u0000\u0000\u00c5\u00cf\u0001"+ + "\u0000\u0000\u0000\u00c6\u00c7\u0003\u0010\b\u0000\u00c7\u00c9\u0005-"+ + "\u0000\u0000\u00c8\u00ca\u00051\u0000\u0000\u00c9\u00c8\u0001\u0000\u0000"+ + "\u0000\u00c9\u00ca\u0001\u0000\u0000\u0000\u00ca\u00cb\u0001\u0000\u0000"+ + "\u0000\u00cb\u00cc\u00052\u0000\u0000\u00cc\u00cf\u0001\u0000\u0000\u0000"+ + "\u00cd\u00cf\u0003\u000e\u0007\u0000\u00ce\u00b1\u0001\u0000\u0000\u0000"+ + "\u00ce\u00b4\u0001\u0000\u0000\u0000\u00ce\u00b5\u0001\u0000\u0000\u0000"+ + "\u00ce\u00b6\u0001\u0000\u0000\u0000\u00ce\u00c6\u0001\u0000\u0000\u0000"+ + "\u00ce\u00cd\u0001\u0000\u0000\u0000\u00cf\u00d8\u0001\u0000\u0000\u0000"+ + "\u00d0\u00d1\n\u0005\u0000\u0000\u00d1\u00d2\u0005\"\u0000\u0000\u00d2"+ + "\u00d7\u0003\n\u0005\u0006\u00d3\u00d4\n\u0004\u0000\u0000\u00d4\u00d5"+ + "\u00054\u0000\u0000\u00d5\u00d7\u0003\n\u0005\u0005\u00d6\u00d0\u0001"+ + "\u0000\u0000\u0000\u00d6\u00d3\u0001\u0000\u0000\u0000\u00d7\u00da\u0001"+ + "\u0000\u0000\u0000\u00d8\u00d6\u0001\u0000\u0000\u0000\u00d8\u00d9\u0001"+ + "\u0000\u0000\u0000\u00d9\u000b\u0001\u0000\u0000\u0000\u00da\u00d8\u0001"+ + "\u0000\u0000\u0000\u00db\u00dd\u0003\u0010\b\u0000\u00dc\u00de\u00051"+ + "\u0000\u0000\u00dd\u00dc\u0001\u0000\u0000\u0000\u00dd\u00de\u0001\u0000"+ + "\u0000\u0000\u00de\u00df\u0001\u0000\u0000\u0000\u00df\u00e0\u0005/\u0000"+ + "\u0000\u00e0\u00e1\u0003j5\u0000\u00e1\u00ea\u0001\u0000\u0000\u0000\u00e2"+ + "\u00e4\u0003\u0010\b\u0000\u00e3\u00e5\u00051\u0000\u0000\u00e4\u00e3"+ + "\u0001\u0000\u0000\u0000\u00e4\u00e5\u0001\u0000\u0000\u0000\u00e5\u00e6"+ + "\u0001\u0000\u0000\u0000\u00e6\u00e7\u00056\u0000\u0000\u00e7\u00e8\u0003"+ + "j5\u0000\u00e8\u00ea\u0001\u0000\u0000\u0000\u00e9\u00db\u0001\u0000\u0000"+ + "\u0000\u00e9\u00e2\u0001\u0000\u0000\u0000\u00ea\r\u0001\u0000\u0000\u0000"+ + "\u00eb\u00ee\u0003:\u001d\u0000\u00ec\u00ed\u0005%\u0000\u0000\u00ed\u00ef"+ + "\u0003\u001a\r\u0000\u00ee\u00ec\u0001\u0000\u0000\u0000\u00ee\u00ef\u0001"+ + "\u0000\u0000\u0000\u00ef\u00f0\u0001\u0000\u0000\u0000\u00f0\u00f1\u0005"+ + "&\u0000\u0000\u00f1\u00f2\u0003D\"\u0000\u00f2\u000f\u0001\u0000\u0000"+ + "\u0000\u00f3\u00f9\u0003\u0012\t\u0000\u00f4\u00f5\u0003\u0012\t\u0000"+ + "\u00f5\u00f6\u0003l6\u0000\u00f6\u00f7\u0003\u0012\t\u0000\u00f7\u00f9"+ + "\u0001\u0000\u0000\u0000\u00f8\u00f3\u0001\u0000\u0000\u0000\u00f8\u00f4"+ + "\u0001\u0000\u0000\u0000\u00f9\u0011\u0001\u0000\u0000\u0000\u00fa\u00fb"+ + "\u0006\t\uffff\uffff\u0000\u00fb\u00ff\u0003\u0014\n\u0000\u00fc\u00fd"+ + "\u0007\u0000\u0000\u0000\u00fd\u00ff\u0003\u0012\t\u0003\u00fe\u00fa\u0001"+ + "\u0000\u0000\u0000\u00fe\u00fc\u0001\u0000\u0000\u0000\u00ff\u0108\u0001"+ + "\u0000\u0000\u0000\u0100\u0101\n\u0002\u0000\u0000\u0101\u0102\u0007\u0001"+ + "\u0000\u0000\u0102\u0107\u0003\u0012\t\u0003\u0103\u0104\n\u0001\u0000"+ + "\u0000\u0104\u0105\u0007\u0000\u0000\u0000\u0105\u0107\u0003\u0012\t\u0002"+ + "\u0106\u0100\u0001\u0000\u0000\u0000\u0106\u0103\u0001\u0000\u0000\u0000"+ + "\u0107\u010a\u0001\u0000\u0000\u0000\u0108\u0106\u0001\u0000\u0000\u0000"+ + "\u0108\u0109\u0001\u0000\u0000\u0000\u0109\u0013\u0001\u0000\u0000\u0000"+ + "\u010a\u0108\u0001\u0000\u0000\u0000\u010b\u010c\u0006\n\uffff\uffff\u0000"+ + "\u010c\u0114\u0003D\"\u0000\u010d\u0114\u0003:\u001d\u0000\u010e\u0114"+ + "\u0003\u0016\u000b\u0000\u010f\u0110\u00050\u0000\u0000\u0110\u0111\u0003"+ + "\n\u0005\u0000\u0111\u0112\u00057\u0000\u0000\u0112\u0114\u0001\u0000"+ + "\u0000\u0000\u0113\u010b\u0001\u0000\u0000\u0000\u0113\u010d\u0001\u0000"+ + "\u0000\u0000\u0113\u010e\u0001\u0000\u0000\u0000\u0113\u010f\u0001\u0000"+ + "\u0000\u0000\u0114\u011a\u0001\u0000\u0000\u0000\u0115\u0116\n\u0001\u0000"+ + "\u0000\u0116\u0117\u0005%\u0000\u0000\u0117\u0119\u0003\u001a\r\u0000"+ + "\u0118\u0115\u0001\u0000\u0000\u0000\u0119\u011c\u0001\u0000\u0000\u0000"+ + "\u011a\u0118\u0001\u0000\u0000\u0000\u011a\u011b\u0001\u0000\u0000\u0000"+ + "\u011b\u0015\u0001\u0000\u0000\u0000\u011c\u011a\u0001\u0000\u0000\u0000"+ + "\u011d\u011e\u0003\u0018\f\u0000\u011e\u0128\u00050\u0000\u0000\u011f"+ + "\u0129\u0005B\u0000\u0000\u0120\u0125\u0003\n\u0005\u0000\u0121\u0122"+ + "\u0005\'\u0000\u0000\u0122\u0124\u0003\n\u0005\u0000\u0123\u0121\u0001"+ + "\u0000\u0000\u0000\u0124\u0127\u0001\u0000\u0000\u0000\u0125\u0123\u0001"+ + "\u0000\u0000\u0000\u0125\u0126\u0001\u0000\u0000\u0000\u0126\u0129\u0001"+ + "\u0000\u0000\u0000\u0127\u0125\u0001\u0000\u0000\u0000\u0128\u011f\u0001"+ + "\u0000\u0000\u0000\u0128\u0120\u0001\u0000\u0000\u0000\u0128\u0129\u0001"+ + "\u0000\u0000\u0000\u0129\u012a\u0001\u0000\u0000\u0000\u012a\u012b\u0005"+ + "7\u0000\u0000\u012b\u0017\u0001\u0000\u0000\u0000\u012c\u012d\u0003H$"+ + "\u0000\u012d\u0019\u0001\u0000\u0000\u0000\u012e\u012f\u0003@ \u0000\u012f"+ + "\u001b\u0001\u0000\u0000\u0000\u0130\u0131\u0005\f\u0000\u0000\u0131\u0132"+ + "\u0003\u001e\u000f\u0000\u0132\u001d\u0001\u0000\u0000\u0000\u0133\u0138"+ + "\u0003 \u0010\u0000\u0134\u0135\u0005\'\u0000\u0000\u0135\u0137\u0003"+ + " \u0010\u0000\u0136\u0134\u0001\u0000\u0000\u0000\u0137\u013a\u0001\u0000"+ + "\u0000\u0000\u0138\u0136\u0001\u0000\u0000\u0000\u0138\u0139\u0001\u0000"+ + "\u0000\u0000\u0139\u001f\u0001\u0000\u0000\u0000\u013a\u0138\u0001\u0000"+ + "\u0000\u0000\u013b\u013c\u0003:\u001d\u0000\u013c\u013d\u0005$\u0000\u0000"+ + "\u013d\u013f\u0001\u0000\u0000\u0000\u013e\u013b\u0001\u0000\u0000\u0000"+ + "\u013e\u013f\u0001\u0000\u0000\u0000\u013f\u0140\u0001\u0000\u0000\u0000"+ + "\u0140\u0141\u0003\n\u0005\u0000\u0141!\u0001\u0000\u0000\u0000\u0142"+ + "\u0143\u0005\u0006\u0000\u0000\u0143\u0148\u0003$\u0012\u0000\u0144\u0145"+ + "\u0005\'\u0000\u0000\u0145\u0147\u0003$\u0012\u0000\u0146\u0144\u0001"+ + "\u0000\u0000\u0000\u0147\u014a\u0001\u0000\u0000\u0000\u0148\u0146\u0001"+ + "\u0000\u0000\u0000\u0148\u0149\u0001\u0000\u0000\u0000\u0149\u014c\u0001"+ + "\u0000\u0000\u0000\u014a\u0148\u0001\u0000\u0000\u0000\u014b\u014d\u0003"+ + "*\u0015\u0000\u014c\u014b\u0001\u0000\u0000\u0000\u014c\u014d\u0001\u0000"+ + "\u0000\u0000\u014d#\u0001\u0000\u0000\u0000\u014e\u014f\u0003&\u0013\u0000"+ + "\u014f\u0150\u0005&\u0000\u0000\u0150\u0152\u0001\u0000\u0000\u0000\u0151"+ + "\u014e\u0001\u0000\u0000\u0000\u0151\u0152\u0001\u0000\u0000\u0000\u0152"+ + "\u0153\u0001\u0000\u0000\u0000\u0153\u0154\u0003(\u0014\u0000\u0154%\u0001"+ + "\u0000\u0000\u0000\u0155\u0156\u0005Q\u0000\u0000\u0156\'\u0001\u0000"+ + "\u0000\u0000\u0157\u0158\u0007\u0002\u0000\u0000\u0158)\u0001\u0000\u0000"+ + "\u0000\u0159\u015c\u0003,\u0016\u0000\u015a\u015c\u0003.\u0017\u0000\u015b"+ + "\u0159\u0001\u0000\u0000\u0000\u015b\u015a\u0001\u0000\u0000\u0000\u015c"+ + "+\u0001\u0000\u0000\u0000\u015d\u015e\u0005P\u0000\u0000\u015e\u0163\u0005"+ + "Q\u0000\u0000\u015f\u0160\u0005\'\u0000\u0000\u0160\u0162\u0005Q\u0000"+ + "\u0000\u0161\u015f\u0001\u0000\u0000\u0000\u0162\u0165\u0001\u0000\u0000"+ + "\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0163\u0164\u0001\u0000\u0000"+ + "\u0000\u0164-\u0001\u0000\u0000\u0000\u0165\u0163\u0001\u0000\u0000\u0000"+ + "\u0166\u0167\u0005F\u0000\u0000\u0167\u0168\u0003,\u0016\u0000\u0168\u0169"+ + "\u0005G\u0000\u0000\u0169/\u0001\u0000\u0000\u0000\u016a\u016b\u0005\u0013"+ + "\u0000\u0000\u016b\u0170\u0003$\u0012\u0000\u016c\u016d\u0005\'\u0000"+ + "\u0000\u016d\u016f\u0003$\u0012\u0000\u016e\u016c\u0001\u0000\u0000\u0000"+ + "\u016f\u0172\u0001\u0000\u0000\u0000\u0170\u016e\u0001\u0000\u0000\u0000"+ + "\u0170\u0171\u0001\u0000\u0000\u0000\u0171\u0174\u0001\u0000\u0000\u0000"+ + "\u0172\u0170\u0001\u0000\u0000\u0000\u0173\u0175\u00036\u001b\u0000\u0174"+ + "\u0173\u0001\u0000\u0000\u0000\u0174\u0175\u0001\u0000\u0000\u0000\u0175"+ + "\u0178\u0001\u0000\u0000\u0000\u0176\u0177\u0005!\u0000\u0000\u0177\u0179"+ + "\u0003\u001e\u000f\u0000\u0178\u0176\u0001\u0000\u0000\u0000\u0178\u0179"+ + "\u0001\u0000\u0000\u0000\u01791\u0001\u0000\u0000\u0000\u017a\u017b\u0005"+ + "\u0004\u0000\u0000\u017b\u017c\u0003\u001e\u000f\u0000\u017c3\u0001\u0000"+ + "\u0000\u0000\u017d\u017f\u0005\u000f\u0000\u0000\u017e\u0180\u00036\u001b"+ + "\u0000\u017f\u017e\u0001\u0000\u0000\u0000\u017f\u0180\u0001\u0000\u0000"+ + "\u0000\u0180\u0183\u0001\u0000\u0000\u0000\u0181\u0182\u0005!\u0000\u0000"+ + "\u0182\u0184\u0003\u001e\u000f\u0000\u0183\u0181\u0001\u0000\u0000\u0000"+ + "\u0183\u0184\u0001\u0000\u0000\u0000\u01845\u0001\u0000\u0000\u0000\u0185"+ + "\u018a\u00038\u001c\u0000\u0186\u0187\u0005\'\u0000\u0000\u0187\u0189"+ + "\u00038\u001c\u0000\u0188\u0186\u0001\u0000\u0000\u0000\u0189\u018c\u0001"+ + "\u0000\u0000\u0000\u018a\u0188\u0001\u0000\u0000\u0000\u018a\u018b\u0001"+ + "\u0000\u0000\u0000\u018b7\u0001\u0000\u0000\u0000\u018c\u018a\u0001\u0000"+ + "\u0000\u0000\u018d\u0190\u0003 \u0010\u0000\u018e\u018f\u0005\u0010\u0000"+ + "\u0000\u018f\u0191\u0003\n\u0005\u0000\u0190\u018e\u0001\u0000\u0000\u0000"+ + "\u0190\u0191\u0001\u0000\u0000\u0000\u01919\u0001\u0000\u0000\u0000\u0192"+ + "\u0197\u0003H$\u0000\u0193\u0194\u0005)\u0000\u0000\u0194\u0196\u0003"+ + "H$\u0000\u0195\u0193\u0001\u0000\u0000\u0000\u0196\u0199\u0001\u0000\u0000"+ + "\u0000\u0197\u0195\u0001\u0000\u0000\u0000\u0197\u0198\u0001\u0000\u0000"+ + "\u0000\u0198;\u0001\u0000\u0000\u0000\u0199\u0197\u0001\u0000\u0000\u0000"+ + "\u019a\u019f\u0003B!\u0000\u019b\u019c\u0005)\u0000\u0000\u019c\u019e"+ + "\u0003B!\u0000\u019d\u019b\u0001\u0000\u0000\u0000\u019e\u01a1\u0001\u0000"+ + "\u0000\u0000\u019f\u019d\u0001\u0000\u0000\u0000\u019f\u01a0\u0001\u0000"+ + "\u0000\u0000\u01a0=\u0001\u0000\u0000\u0000\u01a1\u019f\u0001\u0000\u0000"+ + "\u0000\u01a2\u01a7\u0003<\u001e\u0000\u01a3\u01a4\u0005\'\u0000\u0000"+ + "\u01a4\u01a6\u0003<\u001e\u0000\u01a5\u01a3\u0001\u0000\u0000\u0000\u01a6"+ + "\u01a9\u0001\u0000\u0000\u0000\u01a7\u01a5\u0001\u0000\u0000\u0000\u01a7"+ + "\u01a8\u0001\u0000\u0000\u0000\u01a8?\u0001\u0000\u0000\u0000\u01a9\u01a7"+ + "\u0001\u0000\u0000\u0000\u01aa\u01ab\u0007\u0003\u0000\u0000\u01abA\u0001"+ + "\u0000\u0000\u0000\u01ac\u01b0\u0005U\u0000\u0000\u01ad\u01ae\u0004!\n"+ + "\u0000\u01ae\u01b0\u0003F#\u0000\u01af\u01ac\u0001\u0000\u0000\u0000\u01af"+ + "\u01ad\u0001\u0000\u0000\u0000\u01b0C\u0001\u0000\u0000\u0000\u01b1\u01dc"+ + "\u00052\u0000\u0000\u01b2\u01b3\u0003h4\u0000\u01b3\u01b4\u0005H\u0000"+ + "\u0000\u01b4\u01dc\u0001\u0000\u0000\u0000\u01b5\u01dc\u0003f3\u0000\u01b6"+ + "\u01dc\u0003h4\u0000\u01b7\u01dc\u0003b1\u0000\u01b8\u01dc\u0003F#\u0000"+ + "\u01b9\u01dc\u0003j5\u0000\u01ba\u01bb\u0005F\u0000\u0000\u01bb\u01c0"+ + "\u0003d2\u0000\u01bc\u01bd\u0005\'\u0000\u0000\u01bd\u01bf\u0003d2\u0000"+ + "\u01be\u01bc\u0001\u0000\u0000\u0000\u01bf\u01c2\u0001\u0000\u0000\u0000"+ + "\u01c0\u01be\u0001\u0000\u0000\u0000\u01c0\u01c1\u0001\u0000\u0000\u0000"+ + "\u01c1\u01c3\u0001\u0000\u0000\u0000\u01c2\u01c0\u0001\u0000\u0000\u0000"+ + "\u01c3\u01c4\u0005G\u0000\u0000\u01c4\u01dc\u0001\u0000\u0000\u0000\u01c5"+ + "\u01c6\u0005F\u0000\u0000\u01c6\u01cb\u0003b1\u0000\u01c7\u01c8\u0005"+ + "\'\u0000\u0000\u01c8\u01ca\u0003b1\u0000\u01c9\u01c7\u0001\u0000\u0000"+ + "\u0000\u01ca\u01cd\u0001\u0000\u0000\u0000\u01cb\u01c9\u0001\u0000\u0000"+ + "\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000\u01cc\u01ce\u0001\u0000\u0000"+ + "\u0000\u01cd\u01cb\u0001\u0000\u0000\u0000\u01ce\u01cf\u0005G\u0000\u0000"+ + "\u01cf\u01dc\u0001\u0000\u0000\u0000\u01d0\u01d1\u0005F\u0000\u0000\u01d1"+ + "\u01d6\u0003j5\u0000\u01d2\u01d3\u0005\'\u0000\u0000\u01d3\u01d5\u0003"+ + "j5\u0000\u01d4\u01d2\u0001\u0000\u0000\u0000\u01d5\u01d8\u0001\u0000\u0000"+ + "\u0000\u01d6\u01d4\u0001\u0000\u0000\u0000\u01d6\u01d7\u0001\u0000\u0000"+ + "\u0000\u01d7\u01d9\u0001\u0000\u0000\u0000\u01d8\u01d6\u0001\u0000\u0000"+ + "\u0000\u01d9\u01da\u0005G\u0000\u0000\u01da\u01dc\u0001\u0000\u0000\u0000"+ + "\u01db\u01b1\u0001\u0000\u0000\u0000\u01db\u01b2\u0001\u0000\u0000\u0000"+ + "\u01db\u01b5\u0001\u0000\u0000\u0000\u01db\u01b6\u0001\u0000\u0000\u0000"+ + "\u01db\u01b7\u0001\u0000\u0000\u0000\u01db\u01b8\u0001\u0000\u0000\u0000"+ + "\u01db\u01b9\u0001\u0000\u0000\u0000\u01db\u01ba\u0001\u0000\u0000\u0000"+ + "\u01db\u01c5\u0001\u0000\u0000\u0000\u01db\u01d0\u0001\u0000\u0000\u0000"+ + "\u01dcE\u0001\u0000\u0000\u0000\u01dd\u01e0\u00055\u0000\u0000\u01de\u01e0"+ + "\u0005E\u0000\u0000\u01df\u01dd\u0001\u0000\u0000\u0000\u01df\u01de\u0001"+ + "\u0000\u0000\u0000\u01e0G\u0001\u0000\u0000\u0000\u01e1\u01e5\u0003@ "+ + "\u0000\u01e2\u01e3\u0004$\u000b\u0000\u01e3\u01e5\u0003F#\u0000\u01e4"+ + "\u01e1\u0001\u0000\u0000\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e5"+ + "I\u0001\u0000\u0000\u0000\u01e6\u01e7\u0005\t\u0000\u0000\u01e7\u01e8"+ + "\u0005\u001f\u0000\u0000\u01e8K\u0001\u0000\u0000\u0000\u01e9\u01ea\u0005"+ + "\u000e\u0000\u0000\u01ea\u01ef\u0003N\'\u0000\u01eb\u01ec\u0005\'\u0000"+ + "\u0000\u01ec\u01ee\u0003N\'\u0000\u01ed\u01eb\u0001\u0000\u0000\u0000"+ + "\u01ee\u01f1\u0001\u0000\u0000\u0000\u01ef\u01ed\u0001\u0000\u0000\u0000"+ + "\u01ef\u01f0\u0001\u0000\u0000\u0000\u01f0M\u0001\u0000\u0000\u0000\u01f1"+ + "\u01ef\u0001\u0000\u0000\u0000\u01f2\u01f4\u0003\n\u0005\u0000\u01f3\u01f5"+ + "\u0007\u0004\u0000\u0000\u01f4\u01f3\u0001\u0000\u0000\u0000\u01f4\u01f5"+ + "\u0001\u0000\u0000\u0000\u01f5\u01f8\u0001\u0000\u0000\u0000\u01f6\u01f7"+ + "\u00053\u0000\u0000\u01f7\u01f9\u0007\u0005\u0000\u0000\u01f8\u01f6\u0001"+ + "\u0000\u0000\u0000\u01f8\u01f9\u0001\u0000\u0000\u0000\u01f9O\u0001\u0000"+ + "\u0000\u0000\u01fa\u01fb\u0005\b\u0000\u0000\u01fb\u01fc\u0003>\u001f"+ + "\u0000\u01fcQ\u0001\u0000\u0000\u0000\u01fd\u01fe\u0005\u0002\u0000\u0000"+ + "\u01fe\u01ff\u0003>\u001f\u0000\u01ffS\u0001\u0000\u0000\u0000\u0200\u0201"+ + "\u0005\u000b\u0000\u0000\u0201\u0206\u0003V+\u0000\u0202\u0203\u0005\'"+ + "\u0000\u0000\u0203\u0205\u0003V+\u0000\u0204\u0202\u0001\u0000\u0000\u0000"+ + "\u0205\u0208\u0001\u0000\u0000\u0000\u0206\u0204\u0001\u0000\u0000\u0000"+ + "\u0206\u0207\u0001\u0000\u0000\u0000\u0207U\u0001\u0000\u0000\u0000\u0208"+ + "\u0206\u0001\u0000\u0000\u0000\u0209\u020a\u0003<\u001e\u0000\u020a\u020b"+ + "\u0005Y\u0000\u0000\u020b\u020c\u0003<\u001e\u0000\u020cW\u0001\u0000"+ + "\u0000\u0000\u020d\u020e\u0005\u0001\u0000\u0000\u020e\u020f\u0003\u0014"+ + "\n\u0000\u020f\u0211\u0003j5\u0000\u0210\u0212\u0003^/\u0000\u0211\u0210"+ + "\u0001\u0000\u0000\u0000\u0211\u0212\u0001\u0000\u0000\u0000\u0212Y\u0001"+ + "\u0000\u0000\u0000\u0213\u0214\u0005\u0007\u0000\u0000\u0214\u0215\u0003"+ + "\u0014\n\u0000\u0215\u0216\u0003j5\u0000\u0216[\u0001\u0000\u0000\u0000"+ + "\u0217\u0218\u0005\n\u0000\u0000\u0218\u0219\u0003:\u001d\u0000\u0219"+ + "]\u0001\u0000\u0000\u0000\u021a\u021f\u0003`0\u0000\u021b\u021c\u0005"+ + "\'\u0000\u0000\u021c\u021e\u0003`0\u0000\u021d\u021b\u0001\u0000\u0000"+ + "\u0000\u021e\u0221\u0001\u0000\u0000\u0000\u021f\u021d\u0001\u0000\u0000"+ + "\u0000\u021f\u0220\u0001\u0000\u0000\u0000\u0220_\u0001\u0000\u0000\u0000"+ + "\u0221\u021f\u0001\u0000\u0000\u0000\u0222\u0223\u0003@ \u0000\u0223\u0224"+ + "\u0005$\u0000\u0000\u0224\u0225\u0003D\"\u0000\u0225a\u0001\u0000\u0000"+ + "\u0000\u0226\u0227\u0007\u0006\u0000\u0000\u0227c\u0001\u0000\u0000\u0000"+ + "\u0228\u022b\u0003f3\u0000\u0229\u022b\u0003h4\u0000\u022a\u0228\u0001"+ + "\u0000\u0000\u0000\u022a\u0229\u0001\u0000\u0000\u0000\u022be\u0001\u0000"+ + "\u0000\u0000\u022c\u022e\u0007\u0000\u0000\u0000\u022d\u022c\u0001\u0000"+ + "\u0000\u0000\u022d\u022e\u0001\u0000\u0000\u0000\u022e\u022f\u0001\u0000"+ + "\u0000\u0000\u022f\u0230\u0005 \u0000\u0000\u0230g\u0001\u0000\u0000\u0000"+ + "\u0231\u0233\u0007\u0000\u0000\u0000\u0232\u0231\u0001\u0000\u0000\u0000"+ + "\u0232\u0233\u0001\u0000\u0000\u0000\u0233\u0234\u0001\u0000\u0000\u0000"+ + "\u0234\u0235\u0005\u001f\u0000\u0000\u0235i\u0001\u0000\u0000\u0000\u0236"+ + "\u0237\u0005\u001e\u0000\u0000\u0237k\u0001\u0000\u0000\u0000\u0238\u0239"+ + "\u0007\u0007\u0000\u0000\u0239m\u0001\u0000\u0000\u0000\u023a\u023b\u0005"+ + "\u0005\u0000\u0000\u023b\u023c\u0003p8\u0000\u023co\u0001\u0000\u0000"+ + "\u0000\u023d\u023e\u0005F\u0000\u0000\u023e\u023f\u0003\u0002\u0001\u0000"+ + "\u023f\u0240\u0005G\u0000\u0000\u0240q\u0001\u0000\u0000\u0000\u0241\u0242"+ + "\u0005\r\u0000\u0000\u0242\u0243\u0005i\u0000\u0000\u0243s\u0001\u0000"+ + "\u0000\u0000\u0244\u0245\u0005\u0003\u0000\u0000\u0245\u0248\u0005_\u0000"+ + "\u0000\u0246\u0247\u0005]\u0000\u0000\u0247\u0249\u0003<\u001e\u0000\u0248"+ + "\u0246\u0001\u0000\u0000\u0000\u0248\u0249\u0001\u0000\u0000\u0000\u0249"+ + "\u0253\u0001\u0000\u0000\u0000\u024a\u024b\u0005^\u0000\u0000\u024b\u0250"+ + "\u0003v;\u0000\u024c\u024d\u0005\'\u0000\u0000\u024d\u024f\u0003v;\u0000"+ + "\u024e\u024c\u0001\u0000\u0000\u0000\u024f\u0252\u0001\u0000\u0000\u0000"+ + "\u0250\u024e\u0001\u0000\u0000\u0000\u0250\u0251\u0001\u0000\u0000\u0000"+ + "\u0251\u0254\u0001\u0000\u0000\u0000\u0252\u0250\u0001\u0000\u0000\u0000"+ + "\u0253\u024a\u0001\u0000\u0000\u0000\u0253\u0254\u0001\u0000\u0000\u0000"+ + "\u0254u\u0001\u0000\u0000\u0000\u0255\u0256\u0003<\u001e\u0000\u0256\u0257"+ + "\u0005$\u0000\u0000\u0257\u0259\u0001\u0000\u0000\u0000\u0258\u0255\u0001"+ + "\u0000\u0000\u0000\u0258\u0259\u0001\u0000\u0000\u0000\u0259\u025a\u0001"+ + "\u0000\u0000\u0000\u025a\u025b\u0003<\u001e\u0000\u025bw\u0001\u0000\u0000"+ + "\u0000\u025c\u025d\u0005\u0012\u0000\u0000\u025d\u025e\u0003$\u0012\u0000"+ + "\u025e\u025f\u0005]\u0000\u0000\u025f\u0260\u0003>\u001f\u0000\u0260y"+ + "\u0001\u0000\u0000\u0000\u0261\u0262\u0005\u0011\u0000\u0000\u0262\u0265"+ + "\u00036\u001b\u0000\u0263\u0264\u0005!\u0000\u0000\u0264\u0266\u0003\u001e"+ + "\u000f\u0000\u0265\u0263\u0001\u0000\u0000\u0000\u0265\u0266\u0001\u0000"+ + "\u0000\u0000\u0266{\u0001\u0000\u0000\u0000\u0267\u0269\u0007\b\u0000"+ + "\u0000\u0268\u0267\u0001\u0000\u0000\u0000\u0268\u0269\u0001\u0000\u0000"+ + "\u0000\u0269\u026a\u0001\u0000\u0000\u0000\u026a\u026b\u0005\u0014\u0000"+ + "\u0000\u026b\u026c\u0003~?\u0000\u026c\u026d\u0003\u0080@\u0000\u026d"+ + "}\u0001\u0000\u0000\u0000\u026e\u0271\u0003@ \u0000\u026f\u0270\u0005"+ + "Y\u0000\u0000\u0270\u0272\u0003@ \u0000\u0271\u026f\u0001\u0000\u0000"+ + "\u0000\u0271\u0272\u0001\u0000\u0000\u0000\u0272\u007f\u0001\u0000\u0000"+ + "\u0000\u0273\u0274\u0005]\u0000\u0000\u0274\u0279\u0003\u0082A\u0000\u0275"+ + "\u0276\u0005\'\u0000\u0000\u0276\u0278\u0003\u0082A\u0000\u0277\u0275"+ + "\u0001\u0000\u0000\u0000\u0278\u027b\u0001\u0000\u0000\u0000\u0279\u0277"+ + "\u0001\u0000\u0000\u0000\u0279\u027a\u0001\u0000\u0000\u0000\u027a\u0081"+ + "\u0001\u0000\u0000\u0000\u027b\u0279\u0001\u0000\u0000\u0000\u027c\u027d"+ + "\u0003\u0010\b\u0000\u027d\u0083\u0001\u0000\u0000\u0000>\u008f\u0098"+ + "\u00ac\u00b8\u00c1\u00c9\u00ce\u00d6\u00d8\u00dd\u00e4\u00e9\u00ee\u00f8"+ + "\u00fe\u0106\u0108\u0113\u011a\u0125\u0128\u0138\u013e\u0148\u014c\u0151"+ + "\u015b\u0163\u0170\u0174\u0178\u017f\u0183\u018a\u0190\u0197\u019f\u01a7"+ + "\u01af\u01c0\u01cb\u01d6\u01db\u01df\u01e4\u01ef\u01f4\u01f8\u0206\u0211"+ + "\u021f\u022a\u022d\u0232\u0248\u0250\u0253\u0258\u0265\u0268\u0271\u0279"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index c428a2c6411a1..81d43bc68b79e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -627,13 +627,16 @@ public String visitIdentifierOrParameter(EsqlBaseParser.IdentifierOrParameterCon @Override public Expression visitInlineCast(EsqlBaseParser.InlineCastContext ctx) { - Source source = source(ctx); - DataType dataType = typedParsing(this, ctx.dataType(), DataType.class); + return castToType(source(ctx), ctx.primaryExpression(), ctx.dataType()); + } + + private Expression castToType(Source source, ParseTree parseTree, EsqlBaseParser.DataTypeContext dataTypeCtx) { + DataType dataType = typedParsing(this, dataTypeCtx, DataType.class); var converterToFactory = EsqlDataTypeConverter.converterFunctionFactory(dataType); if (converterToFactory == null) { throw new ParsingException(source, "Unsupported conversion to type [{}]", dataType); } - Expression expr = expression(ctx.primaryExpression()); + Expression expr = expression(parseTree); return converterToFactory.apply(source, expr); } @@ -923,6 +926,14 @@ String unresolvedAttributeNameInParam(ParserRuleContext ctx, Expression param) { @Override public Expression visitMatchBooleanExpression(EsqlBaseParser.MatchBooleanExpressionContext ctx) { - return new Match(source(ctx), expression(ctx.fieldExp), expression(ctx.queryString)); + + final Expression matchFieldExpression; + if (ctx.fieldType != null) { + matchFieldExpression = castToType(source(ctx), ctx.fieldExp, ctx.fieldType); + } else { + matchFieldExpression = expression(ctx.fieldExp); + } + + return new Match(source(ctx), matchFieldExpression, expression(ctx.matchQuery)); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index 1aee8f029e474..7820f0f657f7f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -33,11 +33,13 @@ import org.elasticsearch.xpack.esql.core.querydsl.query.TermsQuery; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.util.Check; import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.fulltext.Term; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils; @@ -533,28 +535,43 @@ private static RangeQuery translate(Range r, TranslatorHandler handler) { public static class MatchFunctionTranslator extends ExpressionTranslator { @Override protected Query asQuery(Match match, TranslatorHandler handler) { - return new MatchQuery(match.source(), ((FieldAttribute) match.field()).name(), match.queryAsText()); + Expression fieldExpression = match.field(); + // Field may be converted to other data type (field_name :: data_type), so we need to check the original field + if (fieldExpression instanceof AbstractConvertFunction convertFunction) { + fieldExpression = convertFunction.field(); + } + if (fieldExpression instanceof FieldAttribute fieldAttribute) { + String fieldName = fieldAttribute.name(); + if (fieldAttribute.field() instanceof MultiTypeEsField multiTypeEsField) { + // If we have multiple field types, we allow the query to be done, but getting the underlying field name + fieldName = multiTypeEsField.getName(); + } + // Make query lenient so mixed field types can be queried when a field type is incompatible with the value provided + return new MatchQuery(match.source(), fieldName, match.queryAsObject(), Map.of("lenient", "true")); + } + + throw new IllegalArgumentException("Match must have a field attribute as the first argument"); } } public static class QueryStringFunctionTranslator extends ExpressionTranslator { @Override protected Query asQuery(QueryString queryString, TranslatorHandler handler) { - return new QueryStringQuery(queryString.source(), queryString.queryAsText(), Map.of(), Map.of()); + return new QueryStringQuery(queryString.source(), (String) queryString.queryAsObject(), Map.of(), Map.of()); } } public static class KqlFunctionTranslator extends ExpressionTranslator { @Override protected Query asQuery(Kql kqlFunction, TranslatorHandler handler) { - return new KqlQuery(kqlFunction.source(), kqlFunction.queryAsText()); + return new KqlQuery(kqlFunction.source(), (String) kqlFunction.queryAsObject()); } } public static class TermFunctionTranslator extends ExpressionTranslator { @Override protected Query asQuery(Term term, TranslatorHandler handler) { - return new TermQuery(term.source(), ((FieldAttribute) term.field()).name(), term.queryAsText()); + return new TermQuery(term.source(), ((FieldAttribute) term.field()).name(), term.queryAsObject()); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 6edbb55af463d..30aec707df541 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -261,10 +261,23 @@ public void testProjectIncludeMultiStarPattern() { } public void testProjectStar() { - assertProjection(""" - from test - | keep * - """, "_meta_field", "emp_no", "first_name", "gender", "job", "job.raw", "languages", "last_name", "long_noidx", "salary"); + assertProjection( + """ + from test + | keep * + """, + "_meta_field", + "emp_no", + "first_name", + "gender", + "hire_date", + "job", + "job.raw", + "languages", + "last_name", + "long_noidx", + "salary" + ); } public void testEscapedStar() { @@ -297,9 +310,22 @@ public void testRenameBacktickPlusPattern() { } public void testNoProjection() { - assertProjection(""" - from test - """, "_meta_field", "emp_no", "first_name", "gender", "job", "job.raw", "languages", "last_name", "long_noidx", "salary"); + assertProjection( + """ + from test + """, + "_meta_field", + "emp_no", + "first_name", + "gender", + "hire_date", + "job", + "job.raw", + "languages", + "last_name", + "long_noidx", + "salary" + ); assertProjectionTypes( """ from test @@ -308,6 +334,7 @@ public void testNoProjection() { DataType.INTEGER, DataType.KEYWORD, DataType.TEXT, + DataType.DATETIME, DataType.TEXT, DataType.KEYWORD, DataType.INTEGER, @@ -329,18 +356,57 @@ public void testDuplicateProjections() { } public void testProjectWildcard() { - assertProjection(""" - from test - | keep first_name, *, last_name - """, "first_name", "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "last_name"); - assertProjection(""" - from test - | keep first_name, last_name, * - """, "first_name", "last_name", "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary"); - assertProjection(""" - from test - | keep *, first_name, last_name - """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "first_name", "last_name"); + assertProjection( + """ + from test + | keep first_name, *, last_name + """, + "first_name", + "_meta_field", + "emp_no", + "gender", + "hire_date", + "job", + "job.raw", + "languages", + "long_noidx", + "salary", + "last_name" + ); + assertProjection( + """ + from test + | keep first_name, last_name, * + """, + "first_name", + "last_name", + "_meta_field", + "emp_no", + "gender", + "hire_date", + "job", + "job.raw", + "languages", + "long_noidx", + "salary" + ); + assertProjection( + """ + from test + | keep *, first_name, last_name + """, + "_meta_field", + "emp_no", + "gender", + "hire_date", + "job", + "job.raw", + "languages", + "long_noidx", + "salary", + "first_name", + "last_name" + ); var e = expectThrows(ParsingException.class, () -> analyze(""" from test @@ -363,22 +429,74 @@ public void testProjectMixedWildcard() { from test | keep *ob*, first_name, *name, first* """, "job", "job.raw", "first_name", "last_name"); - assertProjection(""" - from test - | keep first_name, *, *name - """, "first_name", "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "last_name"); - assertProjection(""" - from test - | keep first*, *, last_name, first_name - """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "last_name", "first_name"); - assertProjection(""" - from test - | keep first*, *, last_name, fir* - """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "last_name", "first_name"); - assertProjection(""" - from test - | keep *, job* - """, "_meta_field", "emp_no", "first_name", "gender", "languages", "last_name", "long_noidx", "salary", "job", "job.raw"); + assertProjection( + """ + from test + | keep first_name, *, *name + """, + "first_name", + "_meta_field", + "emp_no", + "gender", + "hire_date", + "job", + "job.raw", + "languages", + "long_noidx", + "salary", + "last_name" + ); + assertProjection( + """ + from test + | keep first*, *, last_name, first_name + """, + "_meta_field", + "emp_no", + "gender", + "hire_date", + "job", + "job.raw", + "languages", + "long_noidx", + "salary", + "last_name", + "first_name" + ); + assertProjection( + """ + from test + | keep first*, *, last_name, fir* + """, + "_meta_field", + "emp_no", + "gender", + "hire_date", + "job", + "job.raw", + "languages", + "long_noidx", + "salary", + "last_name", + "first_name" + ); + assertProjection( + """ + from test + | keep *, job* + """, + "_meta_field", + "emp_no", + "first_name", + "gender", + "hire_date", + "languages", + "last_name", + "long_noidx", + "salary", + "job", + "job.raw" + ); } public void testProjectThenDropName() { @@ -410,21 +528,34 @@ public void testProjectDropPattern() { from test | keep * | drop *_name - """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary"); + """, "_meta_field", "emp_no", "gender", "hire_date", "job", "job.raw", "languages", "long_noidx", "salary"); } public void testProjectDropNoStarPattern() { assertProjection(""" from test | drop *_name - """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary"); + """, "_meta_field", "emp_no", "gender", "hire_date", "job", "job.raw", "languages", "long_noidx", "salary"); } public void testProjectOrderPatternWithRest() { - assertProjection(""" - from test - | keep *name, *, emp_no - """, "first_name", "last_name", "_meta_field", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "emp_no"); + assertProjection( + """ + from test + | keep *name, *, emp_no + """, + "first_name", + "last_name", + "_meta_field", + "gender", + "hire_date", + "job", + "job.raw", + "languages", + "long_noidx", + "salary", + "emp_no" + ); } public void testProjectDropPatternAndKeepOthers() { @@ -563,7 +694,7 @@ public void testDropPatternUnsupportedFields() { assertProjection(""" from test | drop *ala* - """, "_meta_field", "emp_no", "first_name", "gender", "job", "job.raw", "languages", "last_name", "long_noidx"); + """, "_meta_field", "emp_no", "first_name", "gender", "hire_date", "job", "job.raw", "languages", "last_name", "long_noidx"); } public void testDropUnsupportedPattern() { @@ -633,7 +764,7 @@ public void testRenameReuseAlias() { assertProjection(""" from test | rename emp_no as e, first_name as e - """, "_meta_field", "e", "gender", "job", "job.raw", "languages", "last_name", "long_noidx", "salary"); + """, "_meta_field", "e", "gender", "hire_date", "job", "job.raw", "languages", "last_name", "long_noidx", "salary"); } public void testRenameUnsupportedSubFieldAndResolved() { @@ -1946,6 +2077,7 @@ public void testLookup() { .item(startsWith("emp_no{f}")) .item(startsWith("first_name{f}")) .item(startsWith("gender{f}")) + .item(startsWith("hire_date{f}")) .item(startsWith("job{f}")) .item(startsWith("job.raw{f}")) /* diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 7e3ef4f1f5f87..92cac30f1bb20 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1166,11 +1166,6 @@ public void testMatchInsideEval() throws Exception { } public void testMatchFilter() throws Exception { - assertEquals( - "1:19: first argument of [salary:\"100\"] must be [string], found value [salary] type [integer]", - error("from test | where salary:\"100\"") - ); - assertEquals( "1:19: Invalid condition [first_name:\"Anna\" or starts_with(first_name, \"Anne\")]. " + "[:] operator can't be used as part of an or condition", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index da92eba1e4a05..c609eb3a7ad41 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -739,7 +739,10 @@ public static void testFunctionInfo() { } log.info("{}: tested {} vs annotated {}", arg.name(), signatureTypes, annotationTypes); assertEquals( - "Missmatch between actual and declared parameter types. You probably need to update your @params annotations.", + "Mismatch between actual and declared param type for [" + + arg.name() + + "]. " + + "You probably need to update your @params annotations or add test cases to your test.", signatureTypes, annotationTypes ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchOperatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchOperatorTests.java index 32e9670286ef7..951aff80541bd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchOperatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchOperatorTests.java @@ -10,16 +10,13 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import java.util.LinkedList; -import java.util.List; import java.util.function.Supplier; /** - * This class is only used to generates docs for the match operator - all testing is done in {@link MatchTests} + * This class is only used to generates docs for the match operator - all testing is the same as {@link MatchTests} */ @FunctionName("match_operator") public class MatchOperatorTests extends MatchTests { @@ -30,12 +27,6 @@ public MatchOperatorTests(@Name("TestCase") Supplier @ParametersFactory public static Iterable parameters() { - // Have a minimal test so that we can generate the appropriate types in the docs - List suppliers = new LinkedList<>(); - addPositiveTestCase(List.of(DataType.KEYWORD, DataType.KEYWORD), suppliers); - addPositiveTestCase(List.of(DataType.TEXT, DataType.TEXT), suppliers); - addPositiveTestCase(List.of(DataType.KEYWORD, DataType.TEXT), suppliers); - addPositiveTestCase(List.of(DataType.TEXT, DataType.KEYWORD), suppliers); - return parameterSuppliersFromTypedData(suppliers); + return MatchTests.parameters(); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java index 6a4a7404135f9..f29add60721da 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java @@ -10,119 +10,411 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.NumericUtils; import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; +import java.math.BigInteger; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.function.Supplier; +import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.stringCases; import static org.hamcrest.Matchers.equalTo; @FunctionName("match") public class MatchTests extends AbstractFunctionTestCase { + private static final String FIELD_TYPE_ERROR_STRING = + "keyword, text, boolean, date, date_nanos, double, integer, ip, long, unsigned_long, version"; + + private static final String QUERY_TYPE_ERROR_STRING = + "keyword, boolean, date, date_nanos, double, integer, ip, long, unsigned_long, version"; + public MatchTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { - List> supportedPerPosition = supportedParams(); - List suppliers = new LinkedList<>(); - for (DataType fieldType : DataType.stringTypes()) { - for (DataType queryType : DataType.stringTypes()) { - addPositiveTestCase(List.of(fieldType, queryType), suppliers); - addNonFieldTestCase(List.of(fieldType, queryType), supportedPerPosition, suppliers); - } - } + List suppliers = new ArrayList<>(); - List suppliersWithErrors = errorsForCasesWithoutExamples(suppliers, (v, p) -> "string"); + addUnsignedLongCases(suppliers); + addNumericCases(suppliers); + addNonNumericCases(suppliers); + addQueryAsStringTestCases(suppliers); + addStringTestCases(suppliers); - // Don't test null, as it is not allowed but the expected message is not a type error - so we check it separately in VerifierTests return parameterSuppliersFromTypedData( - suppliersWithErrors.stream().filter(s -> s.types().contains(DataType.NULL) == false).toList() + errorsForCasesWithoutExamples( + suppliers, + (o, v, t) -> errorMessageStringForMatch(o, v, t, (l, p) -> p == 0 ? FIELD_TYPE_ERROR_STRING : QUERY_TYPE_ERROR_STRING) + ) ); } - protected static List> supportedParams() { - Set supportedTextParams = Set.of(DataType.KEYWORD, DataType.TEXT); - Set supportedNumericParams = Set.of(DataType.DOUBLE, DataType.INTEGER); - Set supportedFuzzinessParams = Set.of(DataType.INTEGER, DataType.KEYWORD, DataType.TEXT); - List> supportedPerPosition = List.of( - supportedTextParams, - supportedTextParams, - supportedNumericParams, - supportedFuzzinessParams - ); - return supportedPerPosition; + private static String errorMessageStringForMatch( + boolean includeOrdinal, + List> validPerPosition, + List types, + PositionalErrorMessageSupplier positionalErrorMessageSupplier + ) { + for (int i = 0; i < types.size(); i++) { + // Need to check for nulls and bad parameters in order + if (types.get(i) == DataType.NULL) { + return TypeResolutions.ParamOrdinal.fromIndex(i).name().toLowerCase(Locale.ROOT) + + " argument of [] cannot be null, received [null]"; + } + if (validPerPosition.get(i).contains(types.get(i)) == false) { + break; + } + } + + try { + return typeErrorMessage(includeOrdinal, validPerPosition, types, positionalErrorMessageSupplier); + } catch (IllegalStateException e) { + // This means all the positional args were okay, so the expected error is for nulls or from the combination + return EsqlBinaryComparison.formatIncompatibleTypesMessage(types.get(0), types.get(1), ""); + } } - protected static void addPositiveTestCase(List paramDataTypes, List suppliers) { + private static void addNonNumericCases(List suppliers) { + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.booleanCases(), + TestCaseSupplier.booleanCases(), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.ipCases(), + TestCaseSupplier.ipCases(), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.versionCases(""), + TestCaseSupplier.versionCases(""), + List.of(), + false + ) + ); + // Datetime + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.dateCases(), + TestCaseSupplier.dateCases(), + List.of(), + false + ) + ); - // Positive case - creates an ES field from the field parameter type - suppliers.add( - new TestCaseSupplier( - getTestCaseName(paramDataTypes, "-ES field"), - paramDataTypes, - () -> new TestCaseSupplier.TestCase( - getTestParams(paramDataTypes), - "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]", - DataType.BOOLEAN, - equalTo(true) - ) + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.dateNanosCases(), + TestCaseSupplier.dateNanosCases(), + List.of(), + false ) ); } - private static void addNonFieldTestCase( - List paramDataTypes, - List> supportedPerPosition, - List suppliers - ) { - // Negative case - use directly the field parameter type - suppliers.add( - new TestCaseSupplier( - getTestCaseName(paramDataTypes, "-non ES field"), - paramDataTypes, - typeErrorSupplier(true, supportedPerPosition, paramDataTypes, MatchTests::matchTypeErrorSupplier) + private static void addNumericCases(List suppliers) { + suppliers.addAll( + TestCaseSupplier.forBinaryComparisonWithWidening( + new TestCaseSupplier.NumericTypeTestConfigs<>( + new TestCaseSupplier.NumericTypeTestConfig<>( + (Integer.MIN_VALUE >> 1) - 1, + (Integer.MAX_VALUE >> 1) - 1, + (l, r) -> true, + "EqualsIntsEvaluator" + ), + new TestCaseSupplier.NumericTypeTestConfig<>( + (Long.MIN_VALUE >> 1) - 1, + (Long.MAX_VALUE >> 1) - 1, + (l, r) -> true, + "EqualsLongsEvaluator" + ), + new TestCaseSupplier.NumericTypeTestConfig<>( + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + // NB: this has different behavior than Double::equals + (l, r) -> true, + "EqualsDoublesEvaluator" + ) + ), + "field", + "query", + (lhs, rhs) -> List.of(), + false ) ); } - private static List getTestParams(List paramDataTypes) { - String fieldName = randomIdentifier(); - List params = new ArrayList<>(); - params.add( - new TestCaseSupplier.TypedData( - new FieldExpression(fieldName, List.of(new FieldExpression.FieldValue(fieldName))), - paramDataTypes.get(0), - "field" + private static void addUnsignedLongCases(List suppliers) { + // TODO: These should be integrated into the type cross product above, but are currently broken + // see https://github.com/elastic/elasticsearch/issues/102935 + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true), + TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true), + TestCaseSupplier.intCases(Integer.MIN_VALUE, Integer.MAX_VALUE, true), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true), + TestCaseSupplier.longCases(Long.MIN_VALUE, Long.MAX_VALUE, true), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true), + TestCaseSupplier.doubleCases(Double.MIN_VALUE, Double.MAX_VALUE, true), + List.of(), + false ) ); - params.add(new TestCaseSupplier.TypedData(new BytesRef(randomAlphaOfLength(10)), paramDataTypes.get(1), "query")); - return params; } - private static String getTestCaseName(List paramDataTypes, String fieldType) { - StringBuilder sb = new StringBuilder(); - sb.append("<"); - sb.append(paramDataTypes.get(0)).append(fieldType).append(", "); - sb.append(paramDataTypes.get(1)); - sb.append(">"); - return sb.toString(); + private static void addQueryAsStringTestCases(List suppliers) { + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.intCases(Integer.MIN_VALUE, Integer.MAX_VALUE, true), + TestCaseSupplier.stringCases(DataType.KEYWORD), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.intCases(Integer.MIN_VALUE, Integer.MAX_VALUE, true), + TestCaseSupplier.stringCases(DataType.KEYWORD), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.longCases(Integer.MIN_VALUE, Integer.MAX_VALUE, true), + TestCaseSupplier.stringCases(DataType.KEYWORD), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.doubleCases(Double.MIN_VALUE, Double.MAX_VALUE, true), + TestCaseSupplier.stringCases(DataType.KEYWORD), + List.of(), + false + ) + ); + + // Unsigned Long cases + // TODO: These should be integrated into the type cross product above, but are currently broken + // see https://github.com/elastic/elasticsearch/issues/102935 + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true), + TestCaseSupplier.stringCases(DataType.KEYWORD), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.booleanCases(), + TestCaseSupplier.stringCases(DataType.KEYWORD), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.ipCases(), + TestCaseSupplier.stringCases(DataType.KEYWORD), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.versionCases(""), + TestCaseSupplier.stringCases(DataType.KEYWORD), + List.of(), + false + ) + ); + // Datetime + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.dateCases(), + TestCaseSupplier.stringCases(DataType.KEYWORD), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + null, + "field", + "query", + Object::equals, + DataType.BOOLEAN, + TestCaseSupplier.dateNanosCases(), + TestCaseSupplier.stringCases(DataType.KEYWORD), + List.of(), + false + ) + ); } - private static String matchTypeErrorSupplier(boolean includeOrdinal, List> validPerPosition, List types) { - return "[] cannot operate on [" + types.getFirst().typeName() + "], which is not a field from an index mapping"; + private static void addStringTestCases(List suppliers) { + for (DataType fieldType : DataType.stringTypes()) { + if (DataType.UNDER_CONSTRUCTION.containsKey(fieldType)) { + continue; + } + for (TestCaseSupplier.TypedDataSupplier queryDataSupplier : stringCases(fieldType)) { + suppliers.add( + TestCaseSupplier.testCaseSupplier( + queryDataSupplier, + new TestCaseSupplier.TypedDataSupplier(fieldType.typeName(), () -> randomAlphaOfLength(10), DataType.KEYWORD), + (d1, d2) -> equalTo("string"), + DataType.BOOLEAN, + (o1, o2) -> true + ) + ); + } + } + } + + public final void testLiteralExpressions() { + Expression expression = buildLiteralExpression(testCase); + if (testCase.getExpectedTypeError() != null) { + assertTypeResolutionFailure(expression); + return; + } + assertFalse("expected resolved", expression.typeResolved().unresolved()); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index baef20081a4f2..0c03556241d28 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -372,6 +372,7 @@ public void testMissingFieldInFilterNoProjection() { "emp_no", "first_name", "gender", + "hire_date", "job", "job.raw", "languages", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index d32124c1aaf32..6123a464378f1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -10,6 +10,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.lucene.search.IndexSearcher; +import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; @@ -23,6 +24,7 @@ import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.EsqlTestUtils.TestSearchStats; @@ -40,9 +42,11 @@ import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ExtractAggregateCommonFilter; +import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; @@ -66,14 +70,17 @@ import org.elasticsearch.xpack.esql.stats.Metrics; import org.elasticsearch.xpack.esql.stats.SearchContextStats; import org.elasticsearch.xpack.esql.stats.SearchStats; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; import org.elasticsearch.xpack.kql.query.KqlQueryBuilder; import org.junit.Before; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.BiFunction; import java.util.function.Function; import static java.util.Arrays.asList; @@ -93,12 +100,22 @@ //@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug") public class LocalPhysicalPlanOptimizerTests extends MapperServiceTestCase { + public static final List UNNECESSARY_CASTING_DATA_TYPES = List.of( + DataType.BOOLEAN, + DataType.INTEGER, + DataType.LONG, + DataType.DOUBLE, + DataType.KEYWORD, + DataType.TEXT + ); private static final String PARAM_FORMATTING = "%1$s"; /** * Estimated size of a keyword field in bytes. */ private static final int KEYWORD_EST = EstimatesRowSize.estimateSize(DataType.KEYWORD); + public static final String MATCH_OPERATOR_QUERY = "from test | where %s:%s"; + public static final String MATCH_FUNCTION_QUERY = "from test | where match(%s, %s)"; private TestPlannerOptimizer plannerOptimizer; private final Configuration config; @@ -629,7 +646,7 @@ public void testMatchFunction() { var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); assertThat(query.limit().fold(), is(1000)); - var expected = QueryBuilders.matchQuery("last_name", "Smith"); + var expected = QueryBuilders.matchQuery("last_name", "Smith").lenient(true); assertThat(query.query().toString(), is(expected.toString())); } @@ -661,7 +678,7 @@ public void testMatchFunctionConjunctionWhereOperands() { Source filterSource = new Source(2, 38, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); - var queryString = QueryBuilders.matchQuery("last_name", "Smith"); + var queryString = QueryBuilders.matchQuery("last_name", "Smith").lenient(true); var expected = QueryBuilders.boolQuery().must(queryString).must(range); assertThat(query.query().toString(), is(expected.toString())); } @@ -696,7 +713,7 @@ public void testMatchFunctionWithFunctionsPushedToLucene() { Source filterSource = new Source(2, 32, "cidr_match(ip, \"127.0.0.1/32\")"); var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); - var queryString = QueryBuilders.matchQuery("text", "beta"); + var queryString = QueryBuilders.matchQuery("text", "beta").lenient(true); var expected = QueryBuilders.boolQuery().must(queryString).must(terms); assertThat(query.query().toString(), is(expected.toString())); } @@ -730,7 +747,7 @@ public void testMatchFunctionMultipleWhereClauses() { Source filterSource = new Source(3, 8, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); - var queryString = QueryBuilders.matchQuery("last_name", "Smith"); + var queryString = QueryBuilders.matchQuery("last_name", "Smith").lenient(true); var expected = QueryBuilders.boolQuery().must(queryString).must(range); assertThat(query.query().toString(), is(expected.toString())); } @@ -760,8 +777,8 @@ public void testMatchFunctionMultipleMatchClauses() { var query = as(field.child(), EsQueryExec.class); assertThat(query.limit().fold(), is(1000)); - var queryStringLeft = QueryBuilders.matchQuery("last_name", "Smith"); - var queryStringRight = QueryBuilders.matchQuery("first_name", "John"); + var queryStringLeft = QueryBuilders.matchQuery("last_name", "Smith").lenient(true); + var queryStringRight = QueryBuilders.matchQuery("first_name", "John").lenient(true); var expected = QueryBuilders.boolQuery().must(queryStringLeft).must(queryStringRight); assertThat(query.query().toString(), is(expected.toString())); } @@ -1306,7 +1323,19 @@ public void testMissingFieldsDoNotGetExtracted() { var projections = project.projections(); assertThat( Expressions.names(projections), - contains("_meta_field", "emp_no", "first_name", "gender", "job", "job.raw", "languages", "last_name", "long_noidx", "salary") + contains( + "_meta_field", + "emp_no", + "first_name", + "gender", + "hire_date", + "job", + "job.raw", + "languages", + "last_name", + "long_noidx", + "salary" + ) ); // emp_no assertThat(projections.get(1), instanceOf(ReferenceAttribute.class)); @@ -1314,15 +1343,90 @@ public void testMissingFieldsDoNotGetExtracted() { assertThat(projections.get(2), instanceOf(ReferenceAttribute.class)); // last_name --> first_name - var nullAlias = Alias.unwrap(projections.get(7)); + var nullAlias = Alias.unwrap(projections.get(8)); assertThat(Expressions.name(nullAlias), is("first_name")); // salary --> emp_no - nullAlias = Alias.unwrap(projections.get(9)); + nullAlias = Alias.unwrap(projections.get(10)); assertThat(Expressions.name(nullAlias), is("emp_no")); // check field extraction is skipped and that evaled fields are not extracted anymore var field = as(project.child(), FieldExtractExec.class); var fields = field.attributesToExtract(); - assertThat(Expressions.names(fields), contains("_meta_field", "gender", "job", "job.raw", "languages", "long_noidx")); + assertThat(Expressions.names(fields), contains("_meta_field", "gender", "hire_date", "job", "job.raw", "languages", "long_noidx")); + } + + /* + Checks that match filters are pushed down to Lucene when using no casting, for example: + WHERE first_name:"Anna") + WHERE age:17 + WHERE salary:24.5 + */ + public void testSingleMatchOperatorFilterPushdownWithoutCasting() { + checkMatchFunctionPushDown( + (value, dataType) -> DataType.isString(dataType) ? "\"" + value + "\"" : value.toString(), + value -> value, + UNNECESSARY_CASTING_DATA_TYPES, + MATCH_OPERATOR_QUERY + ); + } + + /* + Checks that match filters are pushed down to Lucene when using strings, for example: + WHERE ip:"127.0.0.1" + WHERE date:"2024-07-01" + WHERE date:"8.17.1" + */ + public void testSingleMatchOperatorFilterPushdownWithStringValues() { + checkMatchFunctionPushDown( + (value, dataType) -> "\"" + value + "\"", + Object::toString, + Match.FIELD_DATA_TYPES, + MATCH_OPERATOR_QUERY + ); + } + + /* + Checks that match filters are pushed down to Lucene when using no casting, for example: + WHERE match(first_name, "Anna") + WHERE match(age, 17) + WHERE match(salary, 24.5) + */ + public void testSingleMatchFunctionFilterPushdownWithoutCasting() { + checkMatchFunctionPushDown( + (value, dataType) -> DataType.isString(dataType) ? "\"" + value + "\"" : value.toString(), + value -> value, + UNNECESSARY_CASTING_DATA_TYPES, + MATCH_FUNCTION_QUERY + ); + } + + /* + Checks that match filters are pushed down to Lucene when using casting, for example: + WHERE match(ip, "127.0.0.1"::IP) + WHERE match(date, "2024-07-01"::DATETIME) + WHERE match(date, "8.17.1"::VERSION) + */ + public void testSingleMatchFunctionPushdownWithCasting() { + checkMatchFunctionPushDown( + LocalPhysicalPlanOptimizerTests::queryValueAsCasting, + value -> value, + Match.FIELD_DATA_TYPES, + MATCH_FUNCTION_QUERY + ); + } + + /* + Checks that match filters are pushed down to Lucene when using strings, for example: + WHERE match(ip, "127.0.0.1") + WHERE match(date, "2024-07-01") + WHERE match(date, "8.17.1") + */ + public void testSingleMatchFunctionFilterPushdownWithStringValues() { + checkMatchFunctionPushDown( + (value, dataType) -> "\"" + value + "\"", + Object::toString, + Match.FIELD_DATA_TYPES, + MATCH_FUNCTION_QUERY + ); } /** @@ -1335,20 +1439,68 @@ public void testMissingFieldsDoNotGetExtracted() { * \_EsQueryExec[test], indexMode[standard], query[{"match":{"first_name":{"query":"Anna"}}}][_doc{f}#13], limit[1000], sort[] * estimatedRowSize[324] */ - public void testSingleMatchFilterPushdown() { - var plan = plannerOptimizer.plan(""" - from test - | where first_name:"Anna" - """); + private void checkMatchFunctionPushDown( + BiFunction queryValueProvider, + Function expectedValueProvider, + Collection fieldDataTypes, + String queryFormat + ) { + var analyzer = makeAnalyzer("mapping-all-types.json"); + // Check for every possible query data type + for (DataType fieldDataType : fieldDataTypes) { + var queryValue = randomQueryValue(fieldDataType); + + String fieldName = fieldDataType == DataType.DATETIME ? "date" : fieldDataType.name().toLowerCase(Locale.ROOT); + var esqlQuery = String.format(Locale.ROOT, queryFormat, fieldName, queryValueProvider.apply(queryValue, fieldDataType)); + + try { + var plan = plannerOptimizer.plan(esqlQuery, IS_SV_STATS, analyzer); + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + var actualLuceneQuery = as(fieldExtract.child(), EsQueryExec.class).query(); + + var expectedLuceneQuery = new MatchQueryBuilder(fieldName, expectedValueProvider.apply(queryValue)).lenient(true); + assertThat("Unexpected match query for data type " + fieldDataType, actualLuceneQuery, equalTo(expectedLuceneQuery)); + } catch (ParsingException e) { + fail("Error parsing ESQL query: " + esqlQuery + "\n" + e.getMessage()); + } + } + } - var limit = as(plan, LimitExec.class); - var exchange = as(limit.child(), ExchangeExec.class); - var project = as(exchange.child(), ProjectExec.class); - var fieldExtract = as(project.child(), FieldExtractExec.class); - var actualLuceneQuery = as(fieldExtract.child(), EsQueryExec.class).query(); + private static Object randomQueryValue(DataType dataType) { + return switch (dataType) { + case BOOLEAN -> randomBoolean(); + case INTEGER -> randomInt(); + case LONG -> randomLong(); + case UNSIGNED_LONG -> randomBigInteger(); + case DATE_NANOS -> EsqlDataTypeConverter.nanoTimeToString(randomMillisUpToYear9999()); + case DATETIME -> EsqlDataTypeConverter.dateTimeToString(randomMillisUpToYear9999()); + case DOUBLE -> randomDouble(); + case KEYWORD -> randomAlphaOfLength(5); + case IP -> NetworkAddress.format(randomIp(randomBoolean())); + case TEXT -> randomAlphaOfLength(50); + case VERSION -> VersionUtils.randomVersion(random()).toString(); + default -> throw new IllegalArgumentException("Unexpected type: " + dataType); + }; + } - var expectedLuceneQuery = new MatchQueryBuilder("first_name", "Anna"); - assertThat(actualLuceneQuery, equalTo(expectedLuceneQuery)); + private static String queryValueAsCasting(Object value, DataType dataType) { + if (value instanceof String) { + value = "\"" + value + "\""; + } + return switch (dataType) { + case VERSION -> value + "::VERSION"; + case IP -> value + "::IP"; + case DATETIME -> value + "::DATETIME"; + case DATE_NANOS -> value + "::DATE_NANOS"; + case INTEGER -> value + "::INTEGER"; + case LONG -> value + "::LONG"; + case BOOLEAN -> String.valueOf(value).toLowerCase(Locale.ROOT); + case UNSIGNED_LONG -> "\"" + value + "\"::UNSIGNED_LONG"; + default -> value.toString(); + }; } /** @@ -1384,10 +1536,10 @@ public void testMultipleMatchFilterPushdown() { var actualLuceneQuery = as(fieldExtract.child(), EsQueryExec.class).query(); Source filterSource = new Source(4, 8, "emp_no > 10000"); - var expectedLuceneQuery = new BoolQueryBuilder().must(new MatchQueryBuilder("first_name", "Anna")) - .must(new MatchQueryBuilder("first_name", "Anneke")) + var expectedLuceneQuery = new BoolQueryBuilder().must(new MatchQueryBuilder("first_name", "Anna").lenient(true)) + .must(new MatchQueryBuilder("first_name", "Anneke").lenient(true)) .must(wrapWithSingleQuery(query, QueryBuilders.rangeQuery("emp_no").gt(10000), "emp_no", filterSource)) - .must(new MatchQueryBuilder("last_name", "Xinglin")); + .must(new MatchQueryBuilder("last_name", "Xinglin").lenient(true)); assertThat(actualLuceneQuery.toString(), is(expectedLuceneQuery.toString())); } @@ -1420,6 +1572,32 @@ public void testTermFunction() { assertThat(query.query().toString(), is(expected.toString())); } + /** + * Expects + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12 + * \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12 + * \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"match":{"emp_no":{"query":123456}}}][_doc{f}#14], + * limit[1000], sort[] estimatedRowSize[332] + */ + public void testMatchWithFieldCasting() { + String query = """ + from test + | where emp_no::long : 123456 + """; + var plan = plannerOptimizer.plan(query); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + var queryExec = as(fieldExtract.child(), EsQueryExec.class); + var queryBuilder = as(queryExec.query(), MatchQueryBuilder.class); + assertThat(queryBuilder.fieldName(), is("emp_no")); + assertThat(queryBuilder.value(), is(123456)); + } + private QueryBuilder wrapWithSingleQuery(String query, QueryBuilder inner, String fieldName, Source source) { return FilterTests.singleValueQuery(query, inner, fieldName, source); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 1f131f79c3d0e..c35f01e9fe774 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -611,6 +611,7 @@ public void testExtractorMultiEvalWithDifferentNames() { "emp_no", "first_name", "gender", + "hire_date", "job", "job.raw", "languages", @@ -652,6 +653,7 @@ public void testExtractorMultiEvalWithSameName() { "emp_no", "first_name", "gender", + "hire_date", "job", "job.raw", "languages", @@ -1172,7 +1174,19 @@ public void testPushLimitAndFilterToSource() { assertThat( names(extract.attributesToExtract()), - contains("_meta_field", "emp_no", "first_name", "gender", "job", "job.raw", "languages", "last_name", "long_noidx", "salary") + contains( + "_meta_field", + "emp_no", + "first_name", + "gender", + "hire_date", + "job", + "job.raw", + "languages", + "last_name", + "long_noidx", + "salary" + ) ); var source = source(extract.child()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 69c00eb395fdb..b83892ea47049 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; @@ -2314,7 +2315,6 @@ public void testInvalidMatchOperator() { "from test | WHERE field:CONCAT(\"hello\", \"world\")", "line 1:25: mismatched input 'CONCAT' expecting {QUOTED_STRING, INTEGER_LITERAL, DECIMAL_LITERAL, " ); - expectError("from test | WHERE field:123::STRING", "line 1:28: mismatched input '::' expecting {, '|', 'and', 'or'}"); expectError( "from test | WHERE field:(true OR false)", "line 1:25: extraneous input '(' expecting {QUOTED_STRING, INTEGER_LITERAL, DECIMAL_LITERAL, " @@ -2323,7 +2323,7 @@ public void testInvalidMatchOperator() { "from test | WHERE field:another_field_or_value", "line 1:25: mismatched input 'another_field_or_value' expecting {QUOTED_STRING, INTEGER_LITERAL, DECIMAL_LITERAL, " ); - expectError("from test | WHERE field:2+3", "line 1:26: mismatched input '+' expecting {, '|', 'and', 'or'}"); + expectError("from test | WHERE field:2+3", "line 1:26: mismatched input '+'"); expectError( "from test | WHERE \"field\":\"value\"", "line 1:26: mismatched input ':' expecting {, '|', 'and', '::', 'or', '+', '-', '*', '/', '%'}" @@ -2333,4 +2333,24 @@ public void testInvalidMatchOperator() { "line 1:37: mismatched input ':' expecting {, '|', 'and', '::', 'or', '+', '-', '*', '/', '%'}" ); } + + public void testMatchFunctionFieldCasting() { + var plan = statement("FROM test | WHERE match(field::int, \"value\")"); + var filter = as(plan, Filter.class); + var function = (UnresolvedFunction) filter.condition(); + var toInteger = (ToInteger) function.children().get(0); + var matchField = (UnresolvedAttribute) toInteger.field(); + assertThat(matchField.name(), equalTo("field")); + assertThat(function.children().get(1).fold(), equalTo("value")); + } + + public void testMatchOperatorFieldCasting() { + var plan = statement("FROM test | WHERE field::int : \"value\""); + var filter = as(plan, Filter.class); + var match = (Match) filter.condition(); + var toInteger = (ToInteger) match.field(); + var matchField = (UnresolvedAttribute) toInteger.field(); + assertThat(matchField.name(), equalTo("field")); + assertThat(match.query().fold(), equalTo("value")); + } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml index 2cd1595d2d5b3..663c0dc78acb3 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml @@ -6,7 +6,6 @@ setup: path: /_query parameters: [ method, path, parameters, capabilities ] capabilities: [ match_operator_colon ] - cluster_features: [ "gte_v8.16.0" ] reason: "Match operator added in 8.16.0" test_runner_features: [capabilities, allowed_warnings_regex] - do: @@ -95,6 +94,25 @@ setup: - length: { values: 1 } - match: { values.0.0: 5 } +--- +"match with integer field": + - requires: + capabilities: + - method: POST + path: /_query + parameters: [ method, path, parameters, capabilities ] + capabilities: [ match_additional_types ] + reason: "Additional types support for match" + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test | WHERE id:3 | KEEP id' + + - length: { values: 1 } + - match: { values.0.0: 3 } + --- "match on non existing column": - do: @@ -178,17 +196,3 @@ setup: - match: { status: 400 } - match: { error.type: verification_exception } - match: { error.reason: "Found 1 problem\nline 1:34: [:] operator is only supported in WHERE commands" } - ---- -"match with non text field": - - do: - catch: bad_request - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - query: 'FROM test | WHERE id:"fox"' - - - match: { status: 400 } - - match: { error.type: verification_exception } - - match: { error.reason: "Found 1 problem\nline 1:19: first argument of [id:\"fox\"] must be [string], found value [id] type [integer]" } From d614804731fc97c468d911ad6e462206103755f0 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Mon, 9 Dec 2024 15:30:14 -0500 Subject: [PATCH 139/204] Remove analyzer version deprecation check (#118167) Version has been deprecated since v7: https://github.com/elastic/elasticsearch/pull/74073 Removing checking for the version setting. It has been ignored and does nothing for the entirety of 8 and for the last minors of v7. --- .../common/ASCIIFoldingTokenFilterFactory.java | 2 +- .../AbstractCompoundWordTokenFilterFactory.java | 2 +- .../common/ApostropheFilterFactory.java | 2 +- .../analysis/common/ArabicAnalyzerProvider.java | 2 +- .../ArabicNormalizationFilterFactory.java | 2 +- .../common/ArabicStemTokenFilterFactory.java | 2 +- .../common/ArmenianAnalyzerProvider.java | 2 +- .../analysis/common/BasqueAnalyzerProvider.java | 2 +- .../common/BengaliAnalyzerProvider.java | 2 +- .../BengaliNormalizationFilterFactory.java | 2 +- .../common/BrazilianAnalyzerProvider.java | 2 +- .../common/BrazilianStemTokenFilterFactory.java | 2 +- .../common/BulgarianAnalyzerProvider.java | 2 +- .../analysis/common/CJKBigramFilterFactory.java | 2 +- .../analysis/common/CJKWidthFilterFactory.java | 2 +- .../common/CatalanAnalyzerProvider.java | 2 +- .../common/CharGroupTokenizerFactory.java | 2 +- .../common/ChineseAnalyzerProvider.java | 2 +- .../analysis/common/CjkAnalyzerProvider.java | 2 +- .../analysis/common/ClassicFilterFactory.java | 2 +- .../common/ClassicTokenizerFactory.java | 2 +- .../common/CommonGramsTokenFilterFactory.java | 2 +- .../analysis/common/CzechAnalyzerProvider.java | 2 +- .../common/CzechStemTokenFilterFactory.java | 2 +- .../analysis/common/DanishAnalyzerProvider.java | 2 +- .../common/DecimalDigitFilterFactory.java | 2 +- .../DelimitedPayloadTokenFilterFactory.java | 2 +- .../analysis/common/DutchAnalyzerProvider.java | 2 +- .../common/DutchStemTokenFilterFactory.java | 2 +- .../common/EdgeNGramTokenFilterFactory.java | 2 +- .../common/EdgeNGramTokenizerFactory.java | 2 +- .../common/ElisionTokenFilterFactory.java | 2 +- .../common/EnglishAnalyzerProvider.java | 2 +- .../common/EstonianAnalyzerProvider.java | 2 +- .../common/FingerprintAnalyzerProvider.java | 2 +- .../common/FingerprintTokenFilterFactory.java | 2 +- .../common/FinnishAnalyzerProvider.java | 2 +- .../common/FlattenGraphTokenFilterFactory.java | 2 +- .../analysis/common/FrenchAnalyzerProvider.java | 2 +- .../common/FrenchStemTokenFilterFactory.java | 2 +- .../common/GalicianAnalyzerProvider.java | 2 +- .../analysis/common/GermanAnalyzerProvider.java | 2 +- .../GermanNormalizationFilterFactory.java | 2 +- .../common/GermanStemTokenFilterFactory.java | 2 +- .../analysis/common/GreekAnalyzerProvider.java | 2 +- .../analysis/common/HindiAnalyzerProvider.java | 2 +- .../common/HindiNormalizationFilterFactory.java | 2 +- .../common/HungarianAnalyzerProvider.java | 2 +- .../common/IndicNormalizationFilterFactory.java | 2 +- .../common/IndonesianAnalyzerProvider.java | 2 +- .../analysis/common/IrishAnalyzerProvider.java | 2 +- .../common/ItalianAnalyzerProvider.java | 2 +- .../common/KStemTokenFilterFactory.java | 2 +- .../analysis/common/KeepTypesFilterFactory.java | 2 +- .../analysis/common/KeepWordFilterFactory.java | 2 +- .../common/KeywordAnalyzerProvider.java | 2 +- .../common/KeywordMarkerTokenFilterFactory.java | 2 +- .../common/KeywordTokenizerFactory.java | 2 +- .../common/LatvianAnalyzerProvider.java | 2 +- .../common/LengthTokenFilterFactory.java | 2 +- .../analysis/common/LetterTokenizerFactory.java | 2 +- .../common/LimitTokenCountFilterFactory.java | 2 +- .../common/LithuanianAnalyzerProvider.java | 2 +- .../common/LowerCaseTokenFilterFactory.java | 2 +- .../common/MinHashTokenFilterFactory.java | 2 +- .../common/MultiplexerTokenFilterFactory.java | 2 +- .../common/NGramTokenFilterFactory.java | 2 +- .../analysis/common/NGramTokenizerFactory.java | 2 +- .../common/NorwegianAnalyzerProvider.java | 2 +- .../common/PathHierarchyTokenizerFactory.java | 2 +- .../common/PatternAnalyzerProvider.java | 2 +- .../PatternCaptureGroupTokenFilterFactory.java | 2 +- .../PatternReplaceTokenFilterFactory.java | 2 +- .../common/PatternTokenizerFactory.java | 2 +- .../common/PersianAnalyzerProvider.java | 2 +- .../PersianNormalizationFilterFactory.java | 2 +- .../common/PersianStemTokenFilterFactory.java | 2 +- .../common/PorterStemTokenFilterFactory.java | 2 +- .../common/PortugueseAnalyzerProvider.java | 2 +- .../PredicateTokenFilterScriptFactory.java | 2 +- .../RemoveDuplicatesTokenFilterFactory.java | 2 +- .../common/ReverseTokenFilterFactory.java | 2 +- .../common/RomanianAnalyzerProvider.java | 2 +- .../common/RussianAnalyzerProvider.java | 2 +- .../common/RussianStemTokenFilterFactory.java | 2 +- .../ScandinavianFoldingFilterFactory.java | 2 +- .../ScandinavianNormalizationFilterFactory.java | 2 +- .../ScriptedConditionTokenFilterFactory.java | 2 +- .../common/SerbianAnalyzerProvider.java | 2 +- .../SerbianNormalizationFilterFactory.java | 2 +- .../analysis/common/SimpleAnalyzerProvider.java | 2 +- .../SimplePatternSplitTokenizerFactory.java | 2 +- .../common/SimplePatternTokenizerFactory.java | 2 +- .../common/SnowballAnalyzerProvider.java | 2 +- .../common/SnowballTokenFilterFactory.java | 2 +- .../analysis/common/SoraniAnalyzerProvider.java | 2 +- .../SoraniNormalizationFilterFactory.java | 2 +- .../common/SpanishAnalyzerProvider.java | 2 +- .../StemmerOverrideTokenFilterFactory.java | 2 +- .../common/StemmerTokenFilterFactory.java | 2 +- .../analysis/common/StopAnalyzerProvider.java | 2 +- .../common/SwedishAnalyzerProvider.java | 2 +- .../common/SynonymTokenFilterFactory.java | 2 +- .../analysis/common/ThaiAnalyzerProvider.java | 2 +- .../analysis/common/ThaiTokenizerFactory.java | 2 +- .../analysis/common/TrimTokenFilterFactory.java | 2 +- .../common/TruncateTokenFilterFactory.java | 2 +- .../common/TurkishAnalyzerProvider.java | 2 +- .../common/UAX29URLEmailTokenizerFactory.java | 2 +- .../common/UniqueTokenFilterFactory.java | 2 +- .../common/UpperCaseTokenFilterFactory.java | 2 +- .../common/WhitespaceAnalyzerProvider.java | 2 +- .../common/WhitespaceTokenizerFactory.java | 2 +- .../WordDelimiterGraphTokenFilterFactory.java | 2 +- .../common/WordDelimiterTokenFilterFactory.java | 2 +- .../common/XLowerCaseTokenizerFactory.java | 2 +- .../analysis/common/CompoundAnalysisTests.java | 4 ---- .../analysis/icu/IcuAnalyzerProvider.java | 2 +- .../icu/IcuCollationTokenFilterFactory.java | 2 +- .../icu/IcuFoldingTokenFilterFactory.java | 2 +- .../icu/IcuNormalizerTokenFilterFactory.java | 2 +- .../analysis/icu/IcuTokenizerFactory.java | 2 +- .../icu/IcuTransformTokenFilterFactory.java | 2 +- .../HiraganaUppercaseFilterFactory.java | 2 +- .../JapaneseStopTokenFilterFactory.java | 2 +- .../KatakanaUppercaseFilterFactory.java | 2 +- .../kuromoji/KuromojiAnalyzerProvider.java | 2 +- .../kuromoji/KuromojiBaseFormFilterFactory.java | 2 +- .../KuromojiCompletionAnalyzerProvider.java | 2 +- .../KuromojiCompletionFilterFactory.java | 2 +- .../KuromojiKatakanaStemmerFactory.java | 2 +- .../kuromoji/KuromojiNumberFilterFactory.java | 2 +- .../KuromojiPartOfSpeechFilterFactory.java | 2 +- .../KuromojiReadingFormFilterFactory.java | 2 +- .../kuromoji/KuromojiTokenizerFactory.java | 2 +- .../analysis/nori/NoriAnalyzerProvider.java | 2 +- .../analysis/nori/NoriNumberFilterFactory.java | 2 +- .../nori/NoriPartOfSpeechStopFilterFactory.java | 2 +- .../nori/NoriReadingFormFilterFactory.java | 2 +- .../analysis/nori/NoriTokenizerFactory.java | 2 +- .../phonetic/PhoneticTokenFilterFactory.java | 2 +- .../smartcn/SmartChineseAnalyzerProvider.java | 2 +- .../SmartChineseNoOpTokenFilterFactory.java | 2 +- .../SmartChineseStopTokenFilterFactory.java | 2 +- .../SmartChineseTokenizerTokenizerFactory.java | 2 +- .../analysis/pl/PolishAnalyzerProvider.java | 2 +- .../pl/PolishStemTokenFilterFactory.java | 2 +- .../pl/PolishStopTokenFilterFactory.java | 2 +- .../ukrainian/UkrainianAnalyzerProvider.java | 2 +- .../indices/recovery/IndexRecoveryIT.java | 17 +++++++---------- .../subphase/highlight/HighlighterSearchIT.java | 9 +++++---- .../analysis/AbstractIndexAnalyzerProvider.java | 4 +--- .../analysis/AbstractTokenFilterFactory.java | 5 +---- .../analysis/AbstractTokenizerFactory.java | 6 +----- .../elasticsearch/index/analysis/Analysis.java | 14 -------------- .../index/analysis/CustomAnalyzerProvider.java | 2 +- .../analysis/CustomNormalizerProvider.java | 2 +- .../analysis/HunspellTokenFilterFactory.java | 2 +- .../analysis/LowercaseNormalizerProvider.java | 2 +- .../analysis/ShingleTokenFilterFactory.java | 2 +- .../analysis/StandardAnalyzerProvider.java | 2 +- .../analysis/StandardTokenizerFactory.java | 2 +- .../index/analysis/StopTokenFilterFactory.java | 2 +- .../indices/analysis/AnalysisModule.java | 2 +- .../indices/TransportAnalyzeActionTests.java | 2 +- .../index/analysis/AnalysisRegistryTests.java | 6 +++--- .../analysis/ReloadableCustomAnalyzerTests.java | 4 ++-- .../index/analysis/StopTokenFilterTests.java | 17 ----------------- .../mapper/TextFieldAnalyzerModeTests.java | 2 +- .../indices/analysis/AnalysisModuleTests.java | 1 - .../analysis/MyFilterTokenFilterFactory.java | 2 +- .../org/elasticsearch/analysis/common/test1.yml | 1 - .../MlClassicTokenizerFactory.java | 2 +- .../MlStandardTokenizerFactory.java | 2 +- 174 files changed, 182 insertions(+), 230 deletions(-) diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java index ba4333bf1be01..bf59fe49d0753 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java @@ -30,7 +30,7 @@ public class ASCIIFoldingTokenFilterFactory extends AbstractTokenFilterFactory i private final boolean preserveOriginal; public ASCIIFoldingTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); preserveOriginal = settings.getAsBoolean(PRESERVE_ORIGINAL.getPreferredName(), DEFAULT_PRESERVE_ORIGINAL); } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AbstractCompoundWordTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AbstractCompoundWordTokenFilterFactory.java index 1b68b2a6ebaf9..c0a8c1374c146 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AbstractCompoundWordTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AbstractCompoundWordTokenFilterFactory.java @@ -30,7 +30,7 @@ public abstract class AbstractCompoundWordTokenFilterFactory extends AbstractTok protected final CharArraySet wordList; protected AbstractCompoundWordTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(name, settings); + super(name); minWordSize = settings.getAsInt("min_word_size", CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE); minSubwordSize = settings.getAsInt("min_subword_size", CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ApostropheFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ApostropheFilterFactory.java index 97a5de3a1408a..28404ac453439 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ApostropheFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ApostropheFilterFactory.java @@ -21,7 +21,7 @@ public class ApostropheFilterFactory extends AbstractTokenFilterFactory { ApostropheFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); } @Override diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicAnalyzerProvider.java index 4e021f8138e76..34869c709a42f 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicAnalyzerProvider.java @@ -22,7 +22,7 @@ public class ArabicAnalyzerProvider extends AbstractIndexAnalyzerProvider asArray = settings.getAsList("ignored_scripts"); Set scripts = new HashSet<>(Arrays.asList("han", "hiragana", "katakana", "hangul")); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKWidthFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKWidthFilterFactory.java index 07e28c5a4924f..717bfc1191ab2 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKWidthFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKWidthFilterFactory.java @@ -20,7 +20,7 @@ public final class CJKWidthFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { CJKWidthFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(name, settings); + super(name); } @Override diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CatalanAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CatalanAnalyzerProvider.java index 2be6f220e4441..b134a1d2ab0fb 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CatalanAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CatalanAnalyzerProvider.java @@ -22,7 +22,7 @@ public class CatalanAnalyzerProvider extends AbstractIndexAnalyzerProvider arrayKeepTypes = settings.getAsList(KEEP_TYPES_KEY, null); if ((arrayKeepTypes == null)) { diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepWordFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepWordFilterFactory.java index 0fa763d627a7a..e2be82df10600 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepWordFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepWordFilterFactory.java @@ -51,7 +51,7 @@ public class KeepWordFilterFactory extends AbstractTokenFilterFactory { private static final String ENABLE_POS_INC_KEY = "enable_position_increments"; KeepWordFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(name, settings); + super(name); final List arrayKeepWords = settings.getAsList(KEEP_WORDS_KEY, null); final String keepWordsPath = settings.get(KEEP_WORDS_PATH_KEY, null); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordAnalyzerProvider.java index c43327cf508cc..c4cd39c4f441f 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordAnalyzerProvider.java @@ -20,7 +20,7 @@ public class KeywordAnalyzerProvider extends AbstractIndexAnalyzerProvider regexes = settings.getAsList(PATTERNS_KEY, null, false); if (regexes == null) { throw new IllegalArgumentException("required setting '" + PATTERNS_KEY + "' is missing for token filter [" + name + "]"); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceTokenFilterFactory.java index cd15b05aabfbe..d90b7a182cab3 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceTokenFilterFactory.java @@ -27,7 +27,7 @@ public class PatternReplaceTokenFilterFactory extends AbstractTokenFilterFactory private final boolean all; public PatternReplaceTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); String sPattern = settings.get("pattern", null); if (sPattern == null) { diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternTokenizerFactory.java index b8f2e194c2ca0..311d63f4d011a 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternTokenizerFactory.java @@ -25,7 +25,7 @@ public class PatternTokenizerFactory extends AbstractTokenizerFactory { private final int group; PatternTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(indexSettings, settings, name); + super(name); String sPattern = settings.get("pattern", "\\W+" /*PatternAnalyzer.NON_WORD_PATTERN*/); if (sPattern == null) { diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianAnalyzerProvider.java index 917a45188123c..b4cb8b8003094 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianAnalyzerProvider.java @@ -35,7 +35,7 @@ public class PersianAnalyzerProvider extends AbstractIndexAnalyzerProvider filterNames; ScriptedConditionTokenFilterFactory(IndexSettings indexSettings, String name, Settings settings, ScriptService scriptService) { - super(name, settings); + super(name); Settings scriptSettings = settings.getAsSettings("script"); Script script = Script.parse(scriptSettings); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianAnalyzerProvider.java index 6dc899be95875..e36aaa9756e70 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianAnalyzerProvider.java @@ -22,7 +22,7 @@ public class SerbianAnalyzerProvider extends AbstractIndexAnalyzerProvider rules = Analysis.getWordList(env, settings, "rules"); if (rules == null) { diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java index 7548c8ad2b88b..a6e9fccd9d09c 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java @@ -92,7 +92,7 @@ public class StemmerTokenFilterFactory extends AbstractTokenFilterFactory { private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(StemmerTokenFilterFactory.class); StemmerTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException { - super(name, settings); + super(name); this.language = Strings.capitalize(settings.get("language", settings.get("name", "porter"))); // check that we have a valid language by trying to create a TokenStream create(EMPTY_TOKEN_STREAM).close(); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StopAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StopAnalyzerProvider.java index 0977d08d0fd48..983a9410fba2d 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StopAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StopAnalyzerProvider.java @@ -23,7 +23,7 @@ public class StopAnalyzerProvider extends AbstractIndexAnalyzerProvider DIGIT diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java index 083594f6ab02e..d20a837e92bf6 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java @@ -47,7 +47,7 @@ public class WordDelimiterTokenFilterFactory extends AbstractTokenFilterFactory @SuppressWarnings("HiddenField") public WordDelimiterTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(name, settings); + super(name); // Sample Format for the type table: // $ => DIGIT diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XLowerCaseTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XLowerCaseTokenizerFactory.java index 7fc1bf6882354..211278d0ece41 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XLowerCaseTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XLowerCaseTokenizerFactory.java @@ -20,7 +20,7 @@ public class XLowerCaseTokenizerFactory extends AbstractTokenizerFactory { public XLowerCaseTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(indexSettings, settings, name); + super(name); } @Override diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CompoundAnalysisTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CompoundAnalysisTests.java index 69dd8e91b52b2..92c134b5c0f56 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CompoundAnalysisTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CompoundAnalysisTests.java @@ -64,7 +64,6 @@ public void testDictionaryDecompounder() throws Exception { hasItems("donau", "dampf", "schiff", "donaudampfschiff", "spargel", "creme", "suppe", "spargelcremesuppe") ); } - assertWarnings("Setting [version] on analysis component [custom7] has no effect and is deprecated"); } public void testHyphenationDecompoundingAnalyzerOnlyLongestMatch() throws Exception { @@ -76,7 +75,6 @@ public void testHyphenationDecompoundingAnalyzerOnlyLongestMatch() throws Except hasItems("kaffeemaschine", "kaffee", "fee", "maschine", "fussballpumpe", "fussball", "ballpumpe", "pumpe") ); } - assertWarnings("Setting [version] on analysis component [custom7] has no effect and is deprecated"); } /** @@ -89,7 +87,6 @@ public void testHyphenationDecompoundingAnalyzerNoSubMatches() throws Exception List terms = analyze(settings, "hyphenationDecompoundingAnalyzerNoSubMatches", "kaffeemaschine fussballpumpe"); MatcherAssert.assertThat(terms, hasItems("kaffeemaschine", "kaffee", "maschine", "fussballpumpe", "fussball", "ballpumpe")); } - assertWarnings("Setting [version] on analysis component [custom7] has no effect and is deprecated"); } /** @@ -102,7 +99,6 @@ public void testHyphenationDecompoundingAnalyzerNoOverlappingMatches() throws Ex List terms = analyze(settings, "hyphenationDecompoundingAnalyzerNoOverlappingMatches", "kaffeemaschine fussballpumpe"); MatcherAssert.assertThat(terms, hasItems("kaffeemaschine", "kaffee", "maschine", "fussballpumpe", "fussball", "pumpe")); } - assertWarnings("Setting [version] on analysis component [custom7] has no effect and is deprecated"); } private List analyze(Settings settings, String analyzerName, String text) throws IOException { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuAnalyzerProvider.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuAnalyzerProvider.java index 3fea1918252ea..9fb611345dbea 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuAnalyzerProvider.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuAnalyzerProvider.java @@ -28,7 +28,7 @@ public class IcuAnalyzerProvider extends AbstractIndexAnalyzerProvider private final Normalizer2 normalizer; public IcuAnalyzerProvider(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); String method = settings.get("method", "nfkc_cf"); String mode = settings.get("mode", "compose"); if ("compose".equals(mode) == false && "decompose".equals(mode) == false) { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java index ae8ead523b7ea..fe0b3a00b2bbb 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java @@ -44,7 +44,7 @@ public class IcuCollationTokenFilterFactory extends AbstractTokenFilterFactory { @SuppressWarnings("HiddenField") public IcuCollationTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); Collator collator; String rules = settings.get("rules"); diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuFoldingTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuFoldingTokenFilterFactory.java index 6cffbb7e0a17e..8932518dc5436 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuFoldingTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuFoldingTokenFilterFactory.java @@ -39,7 +39,7 @@ public class IcuFoldingTokenFilterFactory extends AbstractTokenFilterFactory imp private final Normalizer2 normalizer; public IcuFoldingTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); this.normalizer = IcuNormalizerTokenFilterFactory.wrapWithUnicodeSetFilter(ICU_FOLDING_NORMALIZER, settings); } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuNormalizerTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuNormalizerTokenFilterFactory.java index 23b2c355b7a68..c9eceef30f62e 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuNormalizerTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuNormalizerTokenFilterFactory.java @@ -30,7 +30,7 @@ public class IcuNormalizerTokenFilterFactory extends AbstractTokenFilterFactory private final Normalizer2 normalizer; public IcuNormalizerTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); String method = settings.get("name", "nfkc_cf"); Normalizer2 normalizerInstance = Normalizer2.getInstance(null, method, Normalizer2.Mode.COMPOSE); this.normalizer = wrapWithUnicodeSetFilter(normalizerInstance, settings); diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java index 62ab6d8792905..c66d25ffa2f3b 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java @@ -39,7 +39,7 @@ public class IcuTokenizerFactory extends AbstractTokenizerFactory { private static final String RULE_FILES = "rule_files"; public IcuTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(indexSettings, settings, name); + super(name); config = getIcuConfig(environment, settings); } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTransformTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTransformTokenFilterFactory.java index 785b083c4c31e..5a0a0b3897a47 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTransformTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTransformTokenFilterFactory.java @@ -26,7 +26,7 @@ public class IcuTransformTokenFilterFactory extends AbstractTokenFilterFactory i private final Transliterator transliterator; public IcuTransformTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); this.id = settings.get("id", "Null"); String s = settings.get("dir", "forward"); this.dir = "forward".equals(s) ? Transliterator.FORWARD : Transliterator.REVERSE; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/HiraganaUppercaseFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/HiraganaUppercaseFilterFactory.java index b22757af22372..2d761d99e742f 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/HiraganaUppercaseFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/HiraganaUppercaseFilterFactory.java @@ -18,7 +18,7 @@ public class HiraganaUppercaseFilterFactory extends AbstractTokenFilterFactory { public HiraganaUppercaseFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); } @Override diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/JapaneseStopTokenFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/JapaneseStopTokenFilterFactory.java index e34d5246dd092..3a26e647092f7 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/JapaneseStopTokenFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/JapaneseStopTokenFilterFactory.java @@ -35,7 +35,7 @@ public class JapaneseStopTokenFilterFactory extends AbstractTokenFilterFactory { private final boolean removeTrailing; public JapaneseStopTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(name, settings); + super(name); this.ignoreCase = settings.getAsBoolean("ignore_case", false); this.removeTrailing = settings.getAsBoolean("remove_trailing", true); this.stopWords = Analysis.parseWords( diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KatakanaUppercaseFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KatakanaUppercaseFilterFactory.java index 1f72f1d57d2c5..0776c4ca970a9 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KatakanaUppercaseFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KatakanaUppercaseFilterFactory.java @@ -18,7 +18,7 @@ public class KatakanaUppercaseFilterFactory extends AbstractTokenFilterFactory { public KatakanaUppercaseFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); } @Override diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalyzerProvider.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalyzerProvider.java index db336bec997c4..f0667da992be5 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalyzerProvider.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalyzerProvider.java @@ -26,7 +26,7 @@ public class KuromojiAnalyzerProvider extends AbstractIndexAnalyzerProvider stopWords = Analysis.parseStopWords(env, settings, JapaneseAnalyzer.getDefaultStopSet()); final JapaneseTokenizer.Mode mode = KuromojiTokenizerFactory.getMode(settings); final UserDictionary userDictionary = KuromojiTokenizerFactory.getUserDictionary(env, settings); diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiBaseFormFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiBaseFormFilterFactory.java index 536e6fe993d06..2e8635704c6b0 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiBaseFormFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiBaseFormFilterFactory.java @@ -19,7 +19,7 @@ public class KuromojiBaseFormFilterFactory extends AbstractTokenFilterFactory { public KuromojiBaseFormFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); } @Override diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionAnalyzerProvider.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionAnalyzerProvider.java index c4970251d2bb6..f7121fb211acc 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionAnalyzerProvider.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionAnalyzerProvider.java @@ -22,7 +22,7 @@ public class KuromojiCompletionAnalyzerProvider extends AbstractIndexAnalyzerPro private final JapaneseCompletionAnalyzer analyzer; public KuromojiCompletionAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(name, settings); + super(name); final UserDictionary userDictionary = KuromojiTokenizerFactory.getUserDictionary(env, settings); final Mode mode = KuromojiCompletionFilterFactory.getMode(settings); analyzer = new JapaneseCompletionAnalyzer(userDictionary, mode); diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionFilterFactory.java index 3ec6d08145e69..a7a2d984c3bb7 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionFilterFactory.java @@ -22,7 +22,7 @@ public class KuromojiCompletionFilterFactory extends AbstractTokenFilterFactory private final Mode mode; public KuromojiCompletionFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(name, settings); + super(name); mode = getMode(settings); } diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiKatakanaStemmerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiKatakanaStemmerFactory.java index 7c4d3138381fb..c06f8b7a4941c 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiKatakanaStemmerFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiKatakanaStemmerFactory.java @@ -21,7 +21,7 @@ public class KuromojiKatakanaStemmerFactory extends AbstractTokenFilterFactory { private final int minimumLength; public KuromojiKatakanaStemmerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); minimumLength = settings.getAsInt("minimum_length", JapaneseKatakanaStemFilter.DEFAULT_MINIMUM_LENGTH); } diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiNumberFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiNumberFilterFactory.java index 089b7e10bfae0..605d2c66fac18 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiNumberFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiNumberFilterFactory.java @@ -18,7 +18,7 @@ public class KuromojiNumberFilterFactory extends AbstractTokenFilterFactory { public KuromojiNumberFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); } @Override diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiPartOfSpeechFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiPartOfSpeechFilterFactory.java index e8efa781726f8..5ec3023ca4846 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiPartOfSpeechFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiPartOfSpeechFilterFactory.java @@ -27,7 +27,7 @@ public class KuromojiPartOfSpeechFilterFactory extends AbstractTokenFilterFactor private final Set stopTags = new HashSet<>(); public KuromojiPartOfSpeechFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(name, settings); + super(name); List wordList = Analysis.getWordList(env, settings, "stoptags"); if (wordList != null) { stopTags.addAll(wordList); diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiReadingFormFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiReadingFormFilterFactory.java index 09ab0bbd4b8d8..8e9f03a0c6261 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiReadingFormFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiReadingFormFilterFactory.java @@ -21,7 +21,7 @@ public class KuromojiReadingFormFilterFactory extends AbstractTokenFilterFactory private final boolean useRomaji; public KuromojiReadingFormFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); useRomaji = settings.getAsBoolean("use_romaji", false); } diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java index edb29a8f4c98e..aa978e3e73872 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java @@ -44,7 +44,7 @@ public class KuromojiTokenizerFactory extends AbstractTokenizerFactory { private boolean discardCompoundToken; public KuromojiTokenizerFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(indexSettings, settings, name); + super(name); mode = getMode(settings); userDictionary = getUserDictionary(env, settings); discardPunctuation = settings.getAsBoolean("discard_punctuation", true); diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriAnalyzerProvider.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriAnalyzerProvider.java index 180f6aa0a7f96..36094a4f46df1 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriAnalyzerProvider.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriAnalyzerProvider.java @@ -29,7 +29,7 @@ public class NoriAnalyzerProvider extends AbstractIndexAnalyzerProvider tagList = Analysis.getWordList(env, settings, "stoptags"); diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriNumberFilterFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriNumberFilterFactory.java index cbe92156cd765..ac64c5ff26428 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriNumberFilterFactory.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriNumberFilterFactory.java @@ -19,7 +19,7 @@ public class NoriNumberFilterFactory extends AbstractTokenFilterFactory { public NoriNumberFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); } @Override diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriPartOfSpeechStopFilterFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriPartOfSpeechStopFilterFactory.java index dddb485ab0df3..f93a1bd6e9094 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriPartOfSpeechStopFilterFactory.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriPartOfSpeechStopFilterFactory.java @@ -26,7 +26,7 @@ public class NoriPartOfSpeechStopFilterFactory extends AbstractTokenFilterFactor private final Set stopTags; public NoriPartOfSpeechStopFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(name, settings); + super(name); List tagList = Analysis.getWordList(env, settings, "stoptags"); this.stopTags = tagList != null ? resolvePOSList(tagList) : KoreanPartOfSpeechStopFilter.DEFAULT_STOP_TAGS; } diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriReadingFormFilterFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriReadingFormFilterFactory.java index 1e1b211fdea09..ad5a4a6d1b21a 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriReadingFormFilterFactory.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriReadingFormFilterFactory.java @@ -18,7 +18,7 @@ public class NoriReadingFormFilterFactory extends AbstractTokenFilterFactory { public NoriReadingFormFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); } @Override diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java index ed8458bc94043..40e159b343850 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java @@ -38,7 +38,7 @@ public class NoriTokenizerFactory extends AbstractTokenizerFactory { private final boolean discardPunctuation; public NoriTokenizerFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(indexSettings, settings, name); + super(name); decompoundMode = getMode(settings); userDictionary = getUserDictionary(env, settings, indexSettings); discardPunctuation = settings.getAsBoolean("discard_punctuation", true); diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/PhoneticTokenFilterFactory.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/PhoneticTokenFilterFactory.java index 786c6230349a4..60986fe58db6a 100644 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/PhoneticTokenFilterFactory.java +++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/PhoneticTokenFilterFactory.java @@ -46,7 +46,7 @@ public class PhoneticTokenFilterFactory extends AbstractTokenFilterFactory { private boolean isDaitchMokotoff; public PhoneticTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); this.languageset = null; this.nametype = null; this.ruletype = null; diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseAnalyzerProvider.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseAnalyzerProvider.java index 668b94d0ca972..9dceac60b3c31 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseAnalyzerProvider.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseAnalyzerProvider.java @@ -20,7 +20,7 @@ public class SmartChineseAnalyzerProvider extends AbstractIndexAnalyzerProvider< private final SmartChineseAnalyzer analyzer; public SmartChineseAnalyzerProvider(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); analyzer = new SmartChineseAnalyzer(SmartChineseAnalyzer.getDefaultStopSet()); } diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseNoOpTokenFilterFactory.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseNoOpTokenFilterFactory.java index 55cda67852272..41869b1a7222e 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseNoOpTokenFilterFactory.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseNoOpTokenFilterFactory.java @@ -18,7 +18,7 @@ public class SmartChineseNoOpTokenFilterFactory extends AbstractTokenFilterFactory { public SmartChineseNoOpTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(name, settings); + super(name); } @Override diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseStopTokenFilterFactory.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseStopTokenFilterFactory.java index 2463ad4a2c188..5688971d02865 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseStopTokenFilterFactory.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseStopTokenFilterFactory.java @@ -35,7 +35,7 @@ public class SmartChineseStopTokenFilterFactory extends AbstractTokenFilterFacto private final boolean removeTrailing; public SmartChineseStopTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(name, settings); + super(name); this.ignoreCase = settings.getAsBoolean("ignore_case", false); this.removeTrailing = settings.getAsBoolean("remove_trailing", true); this.stopWords = Analysis.parseWords( diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseTokenizerTokenizerFactory.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseTokenizerTokenizerFactory.java index 2545c9c7d94e8..2b0a9bfe00341 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseTokenizerTokenizerFactory.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseTokenizerTokenizerFactory.java @@ -19,7 +19,7 @@ public class SmartChineseTokenizerTokenizerFactory extends AbstractTokenizerFactory { public SmartChineseTokenizerTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(indexSettings, settings, name); + super(name); } @Override diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalyzerProvider.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalyzerProvider.java index 68e7298473cd4..73f42930e8613 100644 --- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalyzerProvider.java +++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalyzerProvider.java @@ -20,7 +20,7 @@ public class PolishAnalyzerProvider extends AbstractIndexAnalyzerProvider> getTokenFilters() { - return singletonMap( - "test_token_filter", - (indexSettings, environment, name, settings) -> new AbstractTokenFilterFactory(name, settings) { - @Override - public TokenStream create(TokenStream tokenStream) { - if (throwParsingError.get()) { - throw new MapperParsingException("simulate mapping parsing error"); - } - return tokenStream; + return singletonMap("test_token_filter", (indexSettings, environment, name, settings) -> new AbstractTokenFilterFactory(name) { + @Override + public TokenStream create(TokenStream tokenStream) { + if (throwParsingError.get()) { + throw new MapperParsingException("simulate mapping parsing error"); } + return tokenStream; } - ); + }); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java index 36580ebda8aee..fc105d3d4fcd2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java @@ -3703,8 +3703,9 @@ public List getPreConfiguredTokenFilters() { @Override public Map>> getAnalyzers() { - return singletonMap("mock_whitespace", (indexSettings, environment, name, settings) -> { - return new AbstractIndexAnalyzerProvider(name, settings) { + return singletonMap( + "mock_whitespace", + (indexSettings, environment, name, settings) -> new AbstractIndexAnalyzerProvider(name) { MockAnalyzer instance = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); @@ -3712,8 +3713,8 @@ public Map implements AnalyzerProvider { @@ -21,9 +20,8 @@ public abstract class AbstractIndexAnalyzerProvider implemen * * @param name The analyzer name */ - public AbstractIndexAnalyzerProvider(String name, Settings settings) { + public AbstractIndexAnalyzerProvider(String name) { this.name = name; - Analysis.checkForDeprecatedVersion(name, settings); } /** diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AbstractTokenFilterFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/AbstractTokenFilterFactory.java index 13b20a2d42494..7fbc16b54a379 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AbstractTokenFilterFactory.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AbstractTokenFilterFactory.java @@ -9,15 +9,12 @@ package org.elasticsearch.index.analysis; -import org.elasticsearch.common.settings.Settings; - public abstract class AbstractTokenFilterFactory implements TokenFilterFactory { private final String name; - public AbstractTokenFilterFactory(String name, Settings settings) { + public AbstractTokenFilterFactory(String name) { this.name = name; - Analysis.checkForDeprecatedVersion(name, settings); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AbstractTokenizerFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/AbstractTokenizerFactory.java index 4b412003446aa..2b34a45d31b09 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AbstractTokenizerFactory.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AbstractTokenizerFactory.java @@ -9,15 +9,11 @@ package org.elasticsearch.index.analysis; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; - public abstract class AbstractTokenizerFactory implements TokenizerFactory { private final String name; - public AbstractTokenizerFactory(IndexSettings indexSettings, Settings settings, String name) { - Analysis.checkForDeprecatedVersion(name, settings); + public AbstractTokenizerFactory(String name) { this.name = name; } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java b/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java index 462490a7fceb7..505e39a9590ef 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java @@ -50,8 +50,6 @@ import org.apache.lucene.analysis.util.CSVUtil; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.DeprecationCategory; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.synonyms.PagedResult; @@ -80,20 +78,8 @@ public class Analysis { - private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(Analysis.class); private static final Logger logger = LogManager.getLogger(Analysis.class); - public static void checkForDeprecatedVersion(String name, Settings settings) { - String sVersion = settings.get("version"); - if (sVersion != null) { - DEPRECATION_LOGGER.warn( - DeprecationCategory.ANALYSIS, - "analyzer.version", - "Setting [version] on analysis component [" + name + "] has no effect and is deprecated" - ); - } - } - public static CharArraySet parseStemExclusion(Settings settings, CharArraySet defaultStemExclusion) { String value = settings.get("stem_exclusion"); if ("_none_".equals(value)) { diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java index ee6b2e5aaec96..1acf60e19925b 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java @@ -30,7 +30,7 @@ public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider defaultAnalyzers() { .build(); private Analyzer createAnalyzerWithMode(AnalysisMode mode) { - TokenFilterFactory tokenFilter = new AbstractTokenFilterFactory("my_analyzer", Settings.EMPTY) { + TokenFilterFactory tokenFilter = new AbstractTokenFilterFactory("my_analyzer") { @Override public AnalysisMode getAnalysisMode() { return mode; diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index a1b0aedaca2ea..cf6941b84b791 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -113,7 +113,6 @@ public void testSimpleConfigurationJson() throws IOException { public void testSimpleConfigurationYaml() throws IOException { Settings settings = loadFromClasspath("/org/elasticsearch/index/analysis/test1.yml"); testSimpleConfiguration(settings); - assertWarnings("Setting [version] on analysis component [custom7] has no effect and is deprecated"); } private void testSimpleConfiguration(Settings settings) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/analysis/MyFilterTokenFilterFactory.java b/test/framework/src/main/java/org/elasticsearch/index/analysis/MyFilterTokenFilterFactory.java index e6871e720be10..df84a276dc559 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/analysis/MyFilterTokenFilterFactory.java +++ b/test/framework/src/main/java/org/elasticsearch/index/analysis/MyFilterTokenFilterFactory.java @@ -18,7 +18,7 @@ public class MyFilterTokenFilterFactory extends AbstractTokenFilterFactory { public MyFilterTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(name, Settings.EMPTY); + super(name); } @Override diff --git a/test/framework/src/main/resources/org/elasticsearch/analysis/common/test1.yml b/test/framework/src/main/resources/org/elasticsearch/analysis/common/test1.yml index 095b27e0fa071..454b9bc8fc706 100644 --- a/test/framework/src/main/resources/org/elasticsearch/analysis/common/test1.yml +++ b/test/framework/src/main/resources/org/elasticsearch/analysis/common/test1.yml @@ -45,7 +45,6 @@ index : position_increment_gap: 256 custom7 : type : standard - version: 3.6 czechAnalyzerWithStemmer : tokenizer : standard filter : [lowercase, stop, czech_stem] diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlClassicTokenizerFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlClassicTokenizerFactory.java index 0ad3444884f14..e96c72b5d2f0f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlClassicTokenizerFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlClassicTokenizerFactory.java @@ -20,7 +20,7 @@ public class MlClassicTokenizerFactory extends AbstractTokenizerFactory { public MlClassicTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(indexSettings, settings, name); + super(name); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlStandardTokenizerFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlStandardTokenizerFactory.java index 4ceaf4f1967c4..c94816bac81dd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlStandardTokenizerFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlStandardTokenizerFactory.java @@ -23,7 +23,7 @@ public class MlStandardTokenizerFactory extends AbstractTokenizerFactory { public MlStandardTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { - super(indexSettings, settings, name); + super(name); } @Override From 795cd7e08968a16b4381632ab224c05537e7ea81 Mon Sep 17 00:00:00 2001 From: Aditya Singh <72691999+as1605@users.noreply.github.com> Date: Tue, 10 Dec 2024 02:33:43 +0530 Subject: [PATCH 140/204] Fixed variable name in Zstd publish script (#118207) --- dev-tools/publish_zstd_binaries.sh | 4 ++-- docs/changelog/118207.yaml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/118207.yaml diff --git a/dev-tools/publish_zstd_binaries.sh b/dev-tools/publish_zstd_binaries.sh index d5be5c4aaec60..25d4aed6255be 100755 --- a/dev-tools/publish_zstd_binaries.sh +++ b/dev-tools/publish_zstd_binaries.sh @@ -79,8 +79,8 @@ build_linux_jar() { } echo 'Building Linux jars...' -LINUX_ARM_JAR=$(build_linux_jar "linux/amd64" "x86-64") -LINUX_X86_JAR=$(build_linux_jar "linux/arm64" "aarch64") +LINUX_ARM_JAR=$(build_linux_jar "linux/arm64" "aarch64") +LINUX_X86_JAR=$(build_linux_jar "linux/amd64" "x86-64") build_windows_jar() { ARTIFACT="$TEMP/zstd-$VERSION-windows-x86-64.jar" diff --git a/docs/changelog/118207.yaml b/docs/changelog/118207.yaml new file mode 100644 index 0000000000000..989b3c0c5063f --- /dev/null +++ b/docs/changelog/118207.yaml @@ -0,0 +1 @@ +ixed variable name in Zstd publish script for linux platform From 9472489441b00d2460c7b47f52c7754891736e47 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 9 Dec 2024 22:36:16 +0100 Subject: [PATCH 141/204] Upgrade randomized runner to 2.8.2 (#118242) A recent bug we found has been fixed, so the upgrade pulls in the fix and this commit also removes the workaround we put in place. --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 15 ++++++++++----- .../test/AbstractMultiClustersTestCase.java | 7 ------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index aaf654a37dd22..ede1b392b8a41 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -35,7 +35,7 @@ commonscodec = 1.15 protobuf = 3.25.5 # test dependencies -randomizedrunner = 2.8.0 +randomizedrunner = 2.8.2 junit = 4.13.2 junit5 = 5.7.1 hamcrest = 2.1 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 9189d2a27f3f3..33addef8aedd0 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -184,6 +184,11 @@ + + + + + @@ -4478,11 +4483,11 @@ - - - - - + + + + + diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java index 7cd7bce4db187..b4f91f68b8bb7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractMultiClustersTestCase.java @@ -9,8 +9,6 @@ package org.elasticsearch.test; -import com.carrotsearch.randomizedtesting.RandomizedTest; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.cluster.remote.RemoteInfoRequest; @@ -110,11 +108,6 @@ public final void startClusters() throws Exception { MockTransportService.TestPlugin.class, getTestTransportPlugin() ); - // We are going to initialize multiple clusters concurrently, but there is a race condition around the lazy initialization of test - // groups in GroupEvaluator across multiple threads. See https://github.com/randomizedtesting/randomizedtesting/issues/311. - // Calling isNightly before parallelizing is enough to work around that issue. - @SuppressWarnings("unused") - boolean nightly = RandomizedTest.isNightly(); runInParallel(clusterAliases.size(), i -> { String clusterAlias = clusterAliases.get(i); final String clusterName = clusterAlias.equals(LOCAL_CLUSTER) ? "main-cluster" : clusterAlias; From afd88cc5fa0e5b1e18279f7562d9830e1aa64e15 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Mon, 9 Dec 2024 13:58:09 -0800 Subject: [PATCH 142/204] Remove testing for 9.0 unsupported platforms (#118299) --- .buildkite/pipelines/periodic-packaging.template.yml | 5 ----- .buildkite/pipelines/periodic-packaging.yml | 5 ----- .buildkite/pipelines/periodic-platform-support.yml | 6 ------ .buildkite/pipelines/pull-request/packaging-tests-unix.yml | 5 ----- 4 files changed, 21 deletions(-) diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml index 1a1e46d55f7a4..aff0add62a2b6 100644 --- a/.buildkite/pipelines/periodic-packaging.template.yml +++ b/.buildkite/pipelines/periodic-packaging.template.yml @@ -7,19 +7,14 @@ steps: matrix: setup: image: - - debian-11 - debian-12 - opensuse-leap-15 - - oraclelinux-7 - oraclelinux-8 - - sles-12 - sles-15 - - ubuntu-1804 - ubuntu-2004 - ubuntu-2204 - rocky-8 - rocky-9 - - rhel-7 - rhel-8 - rhel-9 - almalinux-8 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index c1b10a46c62a7..9bcd61ac1273c 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -8,19 +8,14 @@ steps: matrix: setup: image: - - debian-11 - debian-12 - opensuse-leap-15 - - oraclelinux-7 - oraclelinux-8 - - sles-12 - sles-15 - - ubuntu-1804 - ubuntu-2004 - ubuntu-2204 - rocky-8 - rocky-9 - - rhel-7 - rhel-8 - rhel-9 - almalinux-8 diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index 79e5a2e8dcdbb..8bee3a78f8316 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -7,19 +7,14 @@ steps: matrix: setup: image: - - debian-11 - debian-12 - opensuse-leap-15 - - oraclelinux-7 - oraclelinux-8 - - sles-12 - sles-15 - - ubuntu-1804 - ubuntu-2004 - ubuntu-2204 - rocky-8 - rocky-9 - - rhel-7 - rhel-8 - rhel-9 - almalinux-8 @@ -90,7 +85,6 @@ steps: setup: image: - amazonlinux-2023 - - amazonlinux-2 agents: provider: aws imagePrefix: elasticsearch-{{matrix.image}} diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml index ffc1350aceab3..6c7dadfd454ed 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml @@ -10,19 +10,14 @@ steps: matrix: setup: image: - - debian-11 - debian-12 - opensuse-leap-15 - - oraclelinux-7 - oraclelinux-8 - - sles-12 - sles-15 - - ubuntu-1804 - ubuntu-2004 - ubuntu-2204 - rocky-8 - rocky-9 - - rhel-7 - rhel-8 - rhel-9 - almalinux-8 From 2be8b6744e4b5837c7221194f3f0b196c2d21855 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Mon, 9 Dec 2024 23:04:22 +0100 Subject: [PATCH 143/204] ESQL: Enable physical plan verification (#118114) This enables the physical plan verification. For it, a couple of changes needed to be applied/corrected: * AggregateMapper creates attributes with unique names; * AggregateExec's verification needs not consider ordinal attribute(s); * LookupJoinExec needs to merge attributes of same name at output, "winning" the right child; * ExchangeExec does no input referencing, since it only outputs all synthetic attributes, "sourced" from remote exchanges; * FieldExtractExec doesn't reference the attributes it "produces". --- docs/changelog/118114.yaml | 5 ++ .../xpack/esql/core/expression/Attribute.java | 5 +- .../optimizer/LocalPhysicalPlanOptimizer.java | 2 +- .../esql/optimizer/PhysicalVerifier.java | 15 +++- .../rules/PlanConsistencyChecker.java | 6 +- .../physical/local/InsertFieldExtraction.java | 17 +---- .../esql/plan/physical/AggregateExec.java | 21 ++++++ .../esql/plan/physical/ExchangeExec.java | 7 ++ .../esql/plan/physical/FieldExtractExec.java | 7 +- .../esql/plan/physical/LookupJoinExec.java | 6 +- .../AbstractPhysicalOperationProviders.java | 4 +- .../xpack/esql/planner/AggregateMapper.java | 62 ++++++++-------- .../LocalPhysicalPlanOptimizerTests.java | 10 +-- .../optimizer/PhysicalPlanOptimizerTests.java | 72 +++++++++++++++++-- 14 files changed, 164 insertions(+), 75 deletions(-) create mode 100644 docs/changelog/118114.yaml diff --git a/docs/changelog/118114.yaml b/docs/changelog/118114.yaml new file mode 100644 index 0000000000000..1b7532d5df981 --- /dev/null +++ b/docs/changelog/118114.yaml @@ -0,0 +1,5 @@ +pr: 118114 +summary: Enable physical plan verification +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java index 53debedafc3d8..829943d245149 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java @@ -49,8 +49,9 @@ public Attribute(Source source, String name, Nullability nullability, @Nullable this.nullability = nullability; } - public static String rawTemporaryName(String inner, String outer, String suffix) { - return SYNTHETIC_ATTRIBUTE_NAME_PREFIX + inner + "$" + outer + "$" + suffix; + public static String rawTemporaryName(String... parts) { + var name = String.join("$", parts); + return name.isEmpty() || name.startsWith(SYNTHETIC_ATTRIBUTE_NAME_PREFIX) ? name : SYNTHETIC_ATTRIBUTE_NAME_PREFIX + name; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java index 48bafd8eef00e..1eaade043658b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java @@ -57,7 +57,7 @@ protected List> batches() { } protected List> rules(boolean optimizeForEsSource) { - List> esSourceRules = new ArrayList<>(4); + List> esSourceRules = new ArrayList<>(6); esSourceRules.add(new ReplaceSourceAttributes()); if (optimizeForEsSource) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalVerifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalVerifier.java index 8bd8aba01fd21..20528f8dc2826 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalVerifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalVerifier.java @@ -8,9 +8,11 @@ package org.elasticsearch.xpack.esql.optimizer; import org.elasticsearch.xpack.esql.common.Failure; +import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.optimizer.rules.PlanConsistencyChecker; +import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; @@ -31,10 +33,14 @@ private PhysicalVerifier() {} /** Verifies the physical plan. */ public Collection verify(PhysicalPlan plan) { Set failures = new LinkedHashSet<>(); + Failures depFailures = new Failures(); plan.forEachDown(p -> { - // FIXME: re-enable - // DEPENDENCY_CHECK.checkPlan(p, failures); + if (p instanceof AggregateExec agg) { + var exclude = Expressions.references(agg.ordinalAttributes()); + DEPENDENCY_CHECK.checkPlan(p, exclude, depFailures); + return; + } if (p instanceof FieldExtractExec fieldExtractExec) { Attribute sourceAttribute = fieldExtractExec.sourceAttribute(); if (sourceAttribute == null) { @@ -48,8 +54,13 @@ public Collection verify(PhysicalPlan plan) { ); } } + DEPENDENCY_CHECK.checkPlan(p, depFailures); }); + if (depFailures.hasFailures()) { + throw new IllegalStateException(depFailures.toString()); + } + return failures; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PlanConsistencyChecker.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PlanConsistencyChecker.java index 30de8945a4c20..5101e3f73bfdf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PlanConsistencyChecker.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PlanConsistencyChecker.java @@ -26,9 +26,13 @@ public class PlanConsistencyChecker

> { * {@link org.elasticsearch.xpack.esql.common.Failure Failure}s to the {@link Failures} object. */ public void checkPlan(P p, Failures failures) { + checkPlan(p, AttributeSet.EMPTY, failures); + } + + public void checkPlan(P p, AttributeSet exclude, Failures failures) { AttributeSet refs = p.references(); AttributeSet input = p.inputSet(); - AttributeSet missing = refs.subtract(input); + AttributeSet missing = refs.subtract(input).subtract(exclude); // TODO: for Joins, we should probably check if the required fields from the left child are actually in the left child, not // just any child (and analogously for the right child). if (missing.isEmpty() == false) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/InsertFieldExtraction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/InsertFieldExtraction.java index ed8851b64c27e..61b1554fb71bc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/InsertFieldExtraction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/InsertFieldExtraction.java @@ -11,7 +11,6 @@ import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; -import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.optimizer.rules.physical.ProjectAwayColumns; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; @@ -22,7 +21,6 @@ import java.util.ArrayList; import java.util.LinkedHashSet; -import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -54,18 +52,9 @@ public PhysicalPlan apply(PhysicalPlan plan) { * it loads the field lazily. If we have more than one field we need to * make sure the fields are loaded for the standard hash aggregator. */ - if (p instanceof AggregateExec agg && agg.groupings().size() == 1) { - // CATEGORIZE requires the standard hash aggregator as well. - if (agg.groupings().get(0).anyMatch(e -> e instanceof Categorize) == false) { - var leaves = new LinkedList<>(); - // TODO: this seems out of place - agg.aggregates() - .stream() - .filter(a -> agg.groupings().contains(a) == false) - .forEach(a -> leaves.addAll(a.collectLeaves())); - var remove = agg.groupings().stream().filter(g -> leaves.contains(g) == false).toList(); - missing.removeAll(Expressions.references(remove)); - } + if (p instanceof AggregateExec agg) { + var ordinalAttributes = agg.ordinalAttributes(); + missing.removeAll(Expressions.references(ordinalAttributes)); } // add extractor diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java index 891d03c571b27..35f45250ed270 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java @@ -18,10 +18,13 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -184,6 +187,24 @@ protected AttributeSet computeReferences() { return mode.isInputPartial() ? new AttributeSet(intermediateAttributes) : Aggregate.computeReferences(aggregates, groupings); } + /** Returns the attributes that can be loaded from ordinals -- no explicit extraction is needed */ + public List ordinalAttributes() { + List orginalAttributs = new ArrayList<>(groupings.size()); + // Ordinals can be leveraged just for a single grouping. If there are multiple groupings, fields need to be laoded for the + // hash aggregator. + // CATEGORIZE requires the standard hash aggregator as well. + if (groupings().size() == 1 && groupings.get(0).anyMatch(e -> e instanceof Categorize) == false) { + var leaves = new HashSet<>(); + aggregates.stream().filter(a -> groupings.contains(a) == false).forEach(a -> leaves.addAll(a.collectLeaves())); + groupings.forEach(g -> { + if (leaves.contains(g) == false) { + orginalAttributs.add((Attribute) g); + } + }); + } + return orginalAttributs; + } + @Override public int hashCode() { return Objects.hash(groupings, aggregates, mode, intermediateAttributes, estimatedRowSize, child()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeExec.java index 5530b3ea54d3d..d1d834b71047a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeExec.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; @@ -72,6 +73,12 @@ public boolean inBetweenAggs() { return inBetweenAggs; } + @Override + protected AttributeSet computeReferences() { + // ExchangeExec does no input referencing, it only outputs all synthetic attributes, "sourced" from remote exchanges. + return AttributeSet.EMPTY; + } + @Override public UnaryExec replaceChild(PhysicalPlan newChild) { return new ExchangeExec(source(), output, inBetweenAggs, newChild); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/FieldExtractExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/FieldExtractExec.java index 35c6e4846bd88..ec996c5c84064 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/FieldExtractExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/FieldExtractExec.java @@ -89,12 +89,7 @@ public static Attribute extractSourceAttributesFrom(PhysicalPlan plan) { @Override protected AttributeSet computeReferences() { - AttributeSet required = new AttributeSet(docValuesAttributes); - - required.add(sourceAttribute); - required.addAll(attributesToExtract); - - return required; + return sourceAttribute != null ? new AttributeSet(sourceAttribute) : AttributeSet.EMPTY; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/LookupJoinExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/LookupJoinExec.java index 8b1cc047309e7..26fd12447e664 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/LookupJoinExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/LookupJoinExec.java @@ -93,9 +93,9 @@ public List addedFields() { public List output() { if (lazyOutput == null) { lazyOutput = new ArrayList<>(left().output()); - for (Attribute attr : addedFields) { - lazyOutput.add(attr); - } + var addedFieldsNames = addedFields.stream().map(Attribute::name).toList(); + lazyOutput.removeIf(a -> addedFieldsNames.contains(a.name())); + lazyOutput.addAll(addedFields); } return lazyOutput; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java index 35aba7665ec87..57ba1c8016feb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java @@ -297,9 +297,9 @@ private void aggregatesToFactory( // coordinator/exchange phase else if (mode == AggregatorMode.FINAL || mode == AggregatorMode.INTERMEDIATE) { if (grouping) { - sourceAttr = aggregateMapper.mapGrouping(aggregateFunction); + sourceAttr = aggregateMapper.mapGrouping(ne); } else { - sourceAttr = aggregateMapper.mapNonGrouping(aggregateFunction); + sourceAttr = aggregateMapper.mapNonGrouping(ne); } } else { throw new EsqlIllegalArgumentException("illegal aggregation mode"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index 18bbfdf485a81..1f55e293b8e75 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -13,6 +13,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeMap; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; @@ -91,7 +92,7 @@ final class AggregateMapper { private record AggDef(Class aggClazz, String type, String extra, boolean grouping) {} /** Map of AggDef types to intermediate named expressions. */ - private static final Map> mapper = AGG_FUNCTIONS.stream() + private static final Map> MAPPER = AGG_FUNCTIONS.stream() .flatMap(AggregateMapper::typeAndNames) .flatMap(AggregateMapper::groupingAndNonGrouping) .collect(Collectors.toUnmodifiableMap(aggDef -> aggDef, AggregateMapper::lookupIntermediateState)); @@ -103,50 +104,57 @@ private record AggDef(Class aggClazz, String type, String extra, boolean grou cache = new HashMap<>(); } - public List mapNonGrouping(List aggregates) { + public List mapNonGrouping(List aggregates) { return doMapping(aggregates, false); } - public List mapNonGrouping(Expression aggregate) { + public List mapNonGrouping(NamedExpression aggregate) { return map(aggregate, false).toList(); } - public List mapGrouping(List aggregates) { + public List mapGrouping(List aggregates) { return doMapping(aggregates, true); } - private List doMapping(List aggregates, boolean grouping) { + private List doMapping(List aggregates, boolean grouping) { AttributeMap attrToExpressions = new AttributeMap<>(); - aggregates.stream().flatMap(agg -> map(agg, grouping)).forEach(ne -> attrToExpressions.put(ne.toAttribute(), ne)); + aggregates.stream().flatMap(ne -> map(ne, grouping)).forEach(ne -> attrToExpressions.put(ne.toAttribute(), ne)); return attrToExpressions.values().stream().toList(); } - public List mapGrouping(Expression aggregate) { + public List mapGrouping(NamedExpression aggregate) { return map(aggregate, true).toList(); } - private Stream map(Expression aggregate, boolean grouping) { - return cache.computeIfAbsent(Alias.unwrap(aggregate), aggKey -> computeEntryForAgg(aggKey, grouping)).stream(); + private Stream map(NamedExpression ne, boolean grouping) { + return cache.computeIfAbsent(Alias.unwrap(ne), aggKey -> computeEntryForAgg(ne.name(), aggKey, grouping)).stream(); } - private static List computeEntryForAgg(Expression aggregate, boolean grouping) { - var aggDef = aggDefOrNull(aggregate, grouping); - if (aggDef != null) { - var is = getNonNull(aggDef); - var exp = isToNE(is).toList(); - return exp; + private static List computeEntryForAgg(String aggAlias, Expression aggregate, boolean grouping) { + if (aggregate instanceof AggregateFunction aggregateFunction) { + return entryForAgg(aggAlias, aggregateFunction, grouping); } if (aggregate instanceof FieldAttribute || aggregate instanceof MetadataAttribute || aggregate instanceof ReferenceAttribute) { - // This condition is a little pedantic, but do we expected other expressions here? if so, then add them + // This condition is a little pedantic, but do we expect other expressions here? if so, then add them return List.of(); - } else { - throw new EsqlIllegalArgumentException("unknown agg: " + aggregate.getClass() + ": " + aggregate); } + throw new EsqlIllegalArgumentException("unknown agg: " + aggregate.getClass() + ": " + aggregate); + } + + private static List entryForAgg(String aggAlias, AggregateFunction aggregateFunction, boolean grouping) { + var aggDef = new AggDef( + aggregateFunction.getClass(), + dataTypeToString(aggregateFunction.field().dataType(), aggregateFunction.getClass()), + aggregateFunction instanceof SpatialCentroid ? "SourceValues" : "", + grouping + ); + var is = getNonNull(aggDef); + return isToNE(is, aggAlias).toList(); } /** Gets the agg from the mapper - wrapper around map::get for more informative failure.*/ private static List getNonNull(AggDef aggDef) { - var l = mapper.get(aggDef); + var l = MAPPER.get(aggDef); if (l == null) { throw new EsqlIllegalArgumentException("Cannot find intermediate state for: " + aggDef); } @@ -199,18 +207,6 @@ private static Stream groupingAndNonGrouping(Tuple, Tuple lookupIntermediateState(AggDef aggDef) { try { @@ -257,7 +253,7 @@ private static String determinePackageName(Class clazz) { } /** Maps intermediate state description to named expressions. */ - private static Stream isToNE(List intermediateStateDescs) { + private static Stream isToNE(List intermediateStateDescs, String aggAlias) { return intermediateStateDescs.stream().map(is -> { final DataType dataType; if (Strings.isEmpty(is.dataType())) { @@ -265,7 +261,7 @@ private static Stream isToNE(List interm } else { dataType = DataType.fromEs(is.dataType()); } - return new ReferenceAttribute(Source.EMPTY, is.name(), dataType); + return new ReferenceAttribute(Source.EMPTY, Attribute.rawTemporaryName(aggAlias, is.name()), dataType); }); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 6123a464378f1..879a413615202 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -255,7 +255,7 @@ public void testCountFieldWithEval() { var esStatsQuery = as(exg.child(), EsStatsQueryExec.class); assertThat(esStatsQuery.limit(), is(nullValue())); - assertThat(Expressions.names(esStatsQuery.output()), contains("count", "seen")); + assertThat(Expressions.names(esStatsQuery.output()), contains("$$c$count", "$$c$seen")); var stat = as(esStatsQuery.stats().get(0), Stat.class); assertThat(stat.query(), is(QueryBuilders.existsQuery("salary"))); } @@ -276,7 +276,7 @@ public void testCountOneFieldWithFilter() { var exchange = as(agg.child(), ExchangeExec.class); var esStatsQuery = as(exchange.child(), EsStatsQueryExec.class); assertThat(esStatsQuery.limit(), is(nullValue())); - assertThat(Expressions.names(esStatsQuery.output()), contains("count", "seen")); + assertThat(Expressions.names(esStatsQuery.output()), contains("$$c$count", "$$c$seen")); var stat = as(esStatsQuery.stats().get(0), Stat.class); Source source = new Source(2, 8, "salary > 1000"); var exists = QueryBuilders.existsQuery("salary"); @@ -386,7 +386,7 @@ public void testAnotherCountAllWithFilter() { var exchange = as(agg.child(), ExchangeExec.class); var esStatsQuery = as(exchange.child(), EsStatsQueryExec.class); assertThat(esStatsQuery.limit(), is(nullValue())); - assertThat(Expressions.names(esStatsQuery.output()), contains("count", "seen")); + assertThat(Expressions.names(esStatsQuery.output()), contains("$$c$count", "$$c$seen")); var source = ((SingleValueQuery.Builder) esStatsQuery.query()).source(); var expected = wrapWithSingleQuery(query, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", source); assertThat(expected.toString(), is(esStatsQuery.query().toString())); @@ -997,7 +997,7 @@ public boolean exists(String field) { var exchange = as(agg.child(), ExchangeExec.class); assertThat(exchange.inBetweenAggs(), is(true)); var localSource = as(exchange.child(), LocalSourceExec.class); - assertThat(Expressions.names(localSource.output()), contains("count", "seen")); + assertThat(Expressions.names(localSource.output()), contains("$$c$count", "$$c$seen")); } /** @@ -1152,7 +1152,7 @@ public void testIsNotNull_TextField_Pushdown_WithCount() { var exg = as(agg.child(), ExchangeExec.class); var esStatsQuery = as(exg.child(), EsStatsQueryExec.class); assertThat(esStatsQuery.limit(), is(nullValue())); - assertThat(Expressions.names(esStatsQuery.output()), contains("count", "seen")); + assertThat(Expressions.names(esStatsQuery.output()), contains("$$c$count", "$$c$seen")); var stat = as(esStatsQuery.stats().get(0), Stat.class); assertThat(stat.query(), is(QueryBuilders.existsQuery("job"))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index c35f01e9fe774..9682bb1c8b076 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.compute.aggregation.AggregatorMode; import org.elasticsearch.core.Tuple; import org.elasticsearch.geometry.Circle; import org.elasticsearch.geometry.Polygon; @@ -127,6 +128,7 @@ import org.elasticsearch.xpack.esql.stats.SearchStats; import org.junit.Before; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -2286,6 +2288,58 @@ public void testFieldExtractWithoutSourceAttributes() { ); } + public void testVerifierOnMissingReferences() { + var plan = physicalPlan(""" + from test + | stats s = sum(salary) by emp_no + | where emp_no > 10 + """); + + plan = plan.transformUp( + AggregateExec.class, + a -> new AggregateExec( + a.source(), + a.child(), + a.groupings(), + List.of(), // remove the aggs (and thus the groupings) entirely + a.getMode(), + a.intermediateAttributes(), + a.estimatedRowSize() + ) + ); + final var finalPlan = plan; + var e = expectThrows(IllegalStateException.class, () -> physicalPlanOptimizer.verify(finalPlan)); + assertThat(e.getMessage(), containsString(" > 10[INTEGER]]] optimized incorrectly due to missing references [emp_no{f}#")); + } + + public void testVerifierOnDuplicateOutputAttributes() { + var plan = physicalPlan(""" + from test + | stats s = sum(salary) by emp_no + | where emp_no > 10 + """); + + plan = plan.transformUp(AggregateExec.class, a -> { + List intermediates = new ArrayList<>(a.intermediateAttributes()); + intermediates.add(intermediates.get(0)); + return new AggregateExec( + a.source(), + a.child(), + a.groupings(), + a.aggregates(), + AggregatorMode.INTERMEDIATE, // FINAL would deduplicate aggregates() + intermediates, + a.estimatedRowSize() + ); + }); + final var finalPlan = plan; + var e = expectThrows(IllegalStateException.class, () -> physicalPlanOptimizer.verify(finalPlan)); + assertThat( + e.getMessage(), + containsString("Plan [LimitExec[1000[INTEGER]]] optimized incorrectly due to duplicate output attribute emp_no{f}#") + ); + } + public void testProjectAwayColumns() { var rule = new ProjectAwayColumns(); @@ -2557,7 +2611,7 @@ public boolean exists(String field) { var exchange = asRemoteExchange(aggregate.child()); var localSourceExec = as(exchange.child(), LocalSourceExec.class); - assertThat(Expressions.names(localSourceExec.output()), contains("languages", "min", "seen")); + assertThat(Expressions.names(localSourceExec.output()), contains("languages", "$$m$min", "$$m$seen")); } /** @@ -2593,9 +2647,9 @@ public void testPartialAggFoldingOutput() { var limit = as(optimized, LimitExec.class); var agg = as(limit.child(), AggregateExec.class); var exchange = as(agg.child(), ExchangeExec.class); - assertThat(Expressions.names(exchange.output()), contains("count", "seen")); + assertThat(Expressions.names(exchange.output()), contains("$$c$count", "$$c$seen")); var source = as(exchange.child(), LocalSourceExec.class); - assertThat(Expressions.names(source.output()), contains("count", "seen")); + assertThat(Expressions.names(source.output()), contains("$$c$count", "$$c$seen")); } /** @@ -2627,7 +2681,7 @@ public void testGlobalAggFoldingOutput() { var aggFinal = as(limit.child(), AggregateExec.class); var aggPartial = as(aggFinal.child(), AggregateExec.class); // The partial aggregation's output is determined via AbstractPhysicalOperationProviders.intermediateAttributes() - assertThat(Expressions.names(aggPartial.output()), contains("count", "seen")); + assertThat(Expressions.names(aggPartial.output()), contains("$$c$count", "$$c$seen")); limit = as(aggPartial.child(), LimitExec.class); var exchange = as(limit.child(), ExchangeExec.class); var project = as(exchange.child(), ProjectExec.class); @@ -2665,9 +2719,15 @@ public void testPartialAggFoldingOutputForSyntheticAgg() { var aggFinal = as(limit.child(), AggregateExec.class); assertThat(aggFinal.output(), hasSize(2)); var exchange = as(aggFinal.child(), ExchangeExec.class); - assertThat(Expressions.names(exchange.output()), contains("sum", "seen", "count", "seen")); + assertThat( + Expressions.names(exchange.output()), + contains("$$SUM$a$0$sum", "$$SUM$a$0$seen", "$$COUNT$a$1$count", "$$COUNT$a$1$seen") + ); var source = as(exchange.child(), LocalSourceExec.class); - assertThat(Expressions.names(source.output()), contains("sum", "seen", "count", "seen")); + assertThat( + Expressions.names(source.output()), + contains("$$SUM$a$0$sum", "$$SUM$a$0$seen", "$$COUNT$a$1$count", "$$COUNT$a$1$seen") + ); } /** From f78ddaedcb0296f857d334a7aad41ea75598c6b3 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Mon, 9 Dec 2024 14:07:16 -0800 Subject: [PATCH 144/204] Delete changelog entry --- docs/changelog/118207.yaml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/changelog/118207.yaml diff --git a/docs/changelog/118207.yaml b/docs/changelog/118207.yaml deleted file mode 100644 index 989b3c0c5063f..0000000000000 --- a/docs/changelog/118207.yaml +++ /dev/null @@ -1 +0,0 @@ -ixed variable name in Zstd publish script for linux platform From 4f253dcfae773a0d0e0f0a7b7798bd066ac4a89d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:24:20 +1100 Subject: [PATCH 145/204] Mute org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormatTests testSingleVectorCase #118306 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index d356dd2f791d1..05831f3fb04bd 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -291,6 +291,9 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=migrate/10_reindex/Test Reindex With Nonexistent Data Stream} issue: https://github.com/elastic/elasticsearch/issues/118274 +- class: org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormatTests + method: testSingleVectorCase + issue: https://github.com/elastic/elasticsearch/issues/118306 # Examples: # From e52d0c9ae4b9584e44dd5b1bbbee142c11999d1c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:44:15 +1100 Subject: [PATCH 146/204] Mute org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests testBottomFieldSort #118214 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 05831f3fb04bd..a57f79e598fe6 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -294,6 +294,9 @@ tests: - class: org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormatTests method: testSingleVectorCase issue: https://github.com/elastic/elasticsearch/issues/118306 +- class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests + method: testBottomFieldSort + issue: https://github.com/elastic/elasticsearch/issues/118214 # Examples: # From 78620c6c1aa97030a024921b17b4dc2f0881e69d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:26:11 +1100 Subject: [PATCH 147/204] Mute org.elasticsearch.xpack.esql.action.CrossClustersEnrichIT testTopNThenEnrichRemote #118307 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index a57f79e598fe6..12b7ea964969d 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -297,6 +297,9 @@ tests: - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests method: testBottomFieldSort issue: https://github.com/elastic/elasticsearch/issues/118214 +- class: org.elasticsearch.xpack.esql.action.CrossClustersEnrichIT + method: testTopNThenEnrichRemote + issue: https://github.com/elastic/elasticsearch/issues/118307 # Examples: # From 34b7e60f7589f0fd01b04a5cf28a088a254ad295 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Tue, 10 Dec 2024 01:32:56 -0500 Subject: [PATCH 148/204] Re-add ResolvedExpression wrapper (#118174) This PR reapplies #114592 along with an update to remove the performance regression introduced with the original change. --- .../TransportClusterSearchShardsAction.java | 3 +- .../indices/resolve/ResolveIndexAction.java | 9 +- .../query/TransportValidateQueryAction.java | 3 +- .../explain/TransportExplainAction.java | 3 +- .../action/search/TransportSearchAction.java | 24 ++- .../search/TransportSearchShardsAction.java | 6 +- .../metadata/IndexNameExpressionResolver.java | 140 ++++++++++------ .../elasticsearch/indices/IndicesService.java | 3 +- .../elasticsearch/search/SearchService.java | 3 +- .../indices/resolve/ResolveIndexTests.java | 15 +- .../IndexNameExpressionResolverTests.java | 74 ++++++--- .../WildcardExpressionResolverTests.java | 152 +++++++++++------- .../indices/IndicesServiceTests.java | 34 ++-- 13 files changed, 305 insertions(+), 164 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java index 9ffef1f178f44..b855f2cee7613 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; @@ -84,7 +85,7 @@ protected void masterOperation( String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request); Map> routingMap = indexNameExpressionResolver.resolveSearchRouting(state, request.routing(), request.indices()); Map indicesAndFilters = new HashMap<>(); - Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); + Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); for (String index : concreteIndices) { final AliasFilter aliasFilter = indicesService.buildAliasFilter(clusterState, index, indicesAndAliases); final String[] aliases = indexNameExpressionResolver.indexAliases( diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index 5c5c71bc002b3..f5c100b7884bb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -565,8 +566,8 @@ static void resolveIndices( if (names.length == 1 && (Metadata.ALL.equals(names[0]) || Regex.isMatchAllPattern(names[0]))) { names = new String[] { "**" }; } - Set resolvedIndexAbstractions = resolver.resolveExpressions(clusterState, indicesOptions, true, names); - for (String s : resolvedIndexAbstractions) { + Set resolvedIndexAbstractions = resolver.resolveExpressions(clusterState, indicesOptions, true, names); + for (ResolvedExpression s : resolvedIndexAbstractions) { enrichIndexAbstraction(clusterState, s, indices, aliases, dataStreams); } indices.sort(Comparator.comparing(ResolvedIndexAbstraction::getName)); @@ -597,12 +598,12 @@ private static void mergeResults( private static void enrichIndexAbstraction( ClusterState clusterState, - String indexAbstraction, + ResolvedExpression indexAbstraction, List indices, List aliases, List dataStreams ) { - IndexAbstraction ia = clusterState.metadata().getIndicesLookup().get(indexAbstraction); + IndexAbstraction ia = clusterState.metadata().getIndicesLookup().get(indexAbstraction.resource()); if (ia != null) { switch (ia.getType()) { case CONCRETE_INDEX -> { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index 4e9830fe0d14e..e01f364712676 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; @@ -133,7 +134,7 @@ protected void doExecute(Task task, ValidateQueryRequest request, ActionListener @Override protected ShardValidateQueryRequest newShardRequest(int numShards, ShardRouting shard, ValidateQueryRequest request) { final ClusterState clusterState = clusterService.state(); - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); final AliasFilter aliasFilter = searchService.buildAliasFilter(clusterState, shard.getIndexName(), indicesAndAliases); return new ShardValidateQueryRequest(shard.shardId(), aliasFilter, request); } diff --git a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java index 9c82d032014f2..84c6df7b8a66f 100644 --- a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java +++ b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -109,7 +110,7 @@ protected boolean resolveIndex(ExplainRequest request) { @Override protected void resolveRequest(ClusterState state, InternalRequest request) { - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(state, request.request().index()); + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(state, request.request().index()); final AliasFilter aliasFilter = searchService.buildAliasFilter(state, request.concreteIndex(), indicesAndAliases); request.request().filteringAlias(aliasFilter); } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 5d1fb46a53cef..ae27406bf396d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.GroupShardsIterator; @@ -111,6 +112,7 @@ import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; +import java.util.stream.Collectors; import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH; import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH; @@ -207,7 +209,7 @@ public TransportSearchAction( private Map buildPerIndexOriginalIndices( ClusterState clusterState, - Set indicesAndAliases, + Set indicesAndAliases, String[] indices, IndicesOptions indicesOptions ) { @@ -215,6 +217,9 @@ private Map buildPerIndexOriginalIndices( var blocks = clusterState.blocks(); // optimization: mostly we do not have any blocks so there's no point in the expensive per-index checking boolean hasBlocks = blocks.global().isEmpty() == false || blocks.indices().isEmpty() == false; + // Get a distinct set of index abstraction names present from the resolved expressions to help with the reverse resolution from + // concrete index to the expression that produced it. + Set indicesAndAliasesResources = indicesAndAliases.stream().map(ResolvedExpression::resource).collect(Collectors.toSet()); for (String index : indices) { if (hasBlocks) { blocks.indexBlockedRaiseException(ClusterBlockLevel.READ, index); @@ -231,8 +236,8 @@ private Map buildPerIndexOriginalIndices( String[] finalIndices = Strings.EMPTY_ARRAY; if (aliases == null || aliases.length == 0 - || indicesAndAliases.contains(index) - || hasDataStreamRef(clusterState, indicesAndAliases, index)) { + || indicesAndAliasesResources.contains(index) + || hasDataStreamRef(clusterState, indicesAndAliasesResources, index)) { finalIndices = new String[] { index }; } if (aliases != null) { @@ -251,7 +256,11 @@ private static boolean hasDataStreamRef(ClusterState clusterState, Set i return indicesAndAliases.contains(ret.getParentDataStream().getName()); } - Map buildIndexAliasFilters(ClusterState clusterState, Set indicesAndAliases, Index[] concreteIndices) { + Map buildIndexAliasFilters( + ClusterState clusterState, + Set indicesAndAliases, + Index[] concreteIndices + ) { final Map aliasFilterMap = new HashMap<>(); for (Index index : concreteIndices) { clusterState.blocks().indexBlockedRaiseException(ClusterBlockLevel.READ, index.getName()); @@ -1236,7 +1245,10 @@ private void executeSearch( } else { final Index[] indices = resolvedIndices.getConcreteLocalIndices(); concreteLocalIndices = Arrays.stream(indices).map(Index::getName).toArray(String[]::new); - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, searchRequest.indices()); + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions( + clusterState, + searchRequest.indices() + ); aliasFilter = buildIndexAliasFilters(clusterState, indicesAndAliases, indices); aliasFilter.putAll(remoteAliasMap); localShardIterators = getLocalShardsIterator( @@ -1835,7 +1847,7 @@ List getLocalShardsIterator( ClusterState clusterState, SearchRequest searchRequest, String clusterAlias, - Set indicesAndAliases, + Set indicesAndAliases, String[] concreteIndices ) { var routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(), searchRequest.indices()); diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java index d8b57972d604f..614a3e9cf22ae 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.index.Index; @@ -127,7 +128,10 @@ public void searchShards(Task task, SearchShardsRequest searchShardsRequest, Act searchService.getRewriteContext(timeProvider::absoluteStartMillis, resolvedIndices, null), listener.delegateFailureAndWrap((delegate, searchRequest) -> { Index[] concreteIndices = resolvedIndices.getConcreteLocalIndices(); - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, searchRequest.indices()); + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions( + clusterState, + searchRequest.indices() + ); final Map aliasFilters = transportSearchAction.buildIndexAliasFilters( clusterState, indicesAndAliases, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 279243eeff7cf..e7914d812e05c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -80,6 +80,13 @@ public IndexNameExpressionResolver(ThreadContext threadContext, SystemIndices sy this.systemIndices = Objects.requireNonNull(systemIndices, "System Indices must not be null"); } + /** + * This represents a resolved expression in the form of the name of a resource in the cluster. + * Soon it will facilitate an index component selector, which will define which part of the resource the expression is targeting. + * @param resource the name of a resource that an expression refers to. + */ + public record ResolvedExpression(String resource) {} + /** * Same as {@link #concreteIndexNames(ClusterState, IndicesOptions, String...)}, but the index expressions and options * are encapsulated in the specified request. @@ -197,8 +204,9 @@ public List dataStreamNames(ClusterState state, IndicesOptions options, getSystemIndexAccessPredicate(), getNetNewSystemIndexPredicate() ); - final Collection expressions = resolveExpressionsToResources(context, indexExpressions); + final Collection expressions = resolveExpressionsToResources(context, indexExpressions); return expressions.stream() + .map(ResolvedExpression::resource) .map(x -> state.metadata().getIndicesLookup().get(x)) .filter(Objects::nonNull) .filter(ia -> ia.getType() == Type.DATA_STREAM) @@ -227,10 +235,11 @@ public IndexAbstraction resolveWriteIndexAbstraction(ClusterState state, DocWrit getNetNewSystemIndexPredicate() ); - final Collection expressions = resolveExpressionsToResources(context, request.index()); + final Collection expressions = resolveExpressionsToResources(context, request.index()); if (expressions.size() == 1) { - IndexAbstraction ia = state.metadata().getIndicesLookup().get(expressions.iterator().next()); + ResolvedExpression resolvedExpression = expressions.iterator().next(); + IndexAbstraction ia = state.metadata().getIndicesLookup().get(resolvedExpression.resource()); if (ia.getType() == Type.ALIAS) { Index writeIndex = ia.getWriteIndex(); if (writeIndex == null) { @@ -257,7 +266,7 @@ public IndexAbstraction resolveWriteIndexAbstraction(ClusterState state, DocWrit * If {@param preserveDataStreams} is {@code true}, data streams that are covered by the wildcards from the * {@param expressions} are returned as-is, without expanding them further to their respective backing indices. */ - protected static Collection resolveExpressionsToResources(Context context, String... expressions) { + protected static Collection resolveExpressionsToResources(Context context, String... expressions) { // If we do not expand wildcards, then empty or _all expression result in an empty list boolean expandWildcards = context.getOptions().expandWildcardExpressions(); if (expandWildcards == false) { @@ -275,7 +284,7 @@ protected static Collection resolveExpressionsToResources(Context contex } // Using ArrayList when we know we do not have wildcards is an optimisation, given that one expression result in 0 or 1 resources. - Collection resources = expandWildcards && WildcardExpressionResolver.hasWildcards(expressions) + Collection resources = expandWildcards && WildcardExpressionResolver.hasWildcards(expressions) ? new LinkedHashSet<>() : new ArrayList<>(expressions.length); boolean wildcardSeen = false; @@ -297,7 +306,7 @@ protected static Collection resolveExpressionsToResources(Context contex wildcardSeen |= isWildcard; if (isWildcard) { - Set matchingResources = WildcardExpressionResolver.matchWildcardToResources(context, baseExpression); + Set matchingResources = WildcardExpressionResolver.matchWildcardToResources(context, baseExpression); if (context.getOptions().allowNoIndices() == false && matchingResources.isEmpty()) { throw notFoundException(baseExpression); @@ -310,9 +319,9 @@ protected static Collection resolveExpressionsToResources(Context contex } } else { if (isExclusion) { - resources.remove(baseExpression); + resources.remove(new ResolvedExpression(baseExpression)); } else if (ensureAliasOrIndexExists(context, baseExpression)) { - resources.add(baseExpression); + resources.add(new ResolvedExpression(baseExpression)); } } } @@ -428,12 +437,12 @@ String[] concreteIndexNames(Context context, String... indexExpressions) { } Index[] concreteIndices(Context context, String... indexExpressions) { - final Collection expressions = resolveExpressionsToResources(context, indexExpressions); + final Collection expressions = resolveExpressionsToResources(context, indexExpressions); final Set concreteIndicesResult = Sets.newLinkedHashSetWithExpectedSize(expressions.size()); final Map indicesLookup = context.getState().metadata().getIndicesLookup(); - for (String expression : expressions) { - final IndexAbstraction indexAbstraction = indicesLookup.get(expression); + for (ResolvedExpression expression : expressions) { + final IndexAbstraction indexAbstraction = indicesLookup.get(expression.resource()); assert indexAbstraction != null; if (indexAbstraction.getType() == Type.ALIAS && context.isResolveToWriteIndex()) { Index writeIndex = indexAbstraction.getWriteIndex(); @@ -467,7 +476,7 @@ Index[] concreteIndices(Context context, String... indexExpressions) { throw new IllegalArgumentException( indexAbstraction.getType().getDisplayName() + " [" - + expression + + expression.resource() + "] has more than one index associated with it " + Arrays.toString(indexNames) + ", can't execute a single index op" @@ -682,7 +691,7 @@ public Index concreteSingleIndex(ClusterState state, IndicesRequest request) { * Utility method that allows to resolve an index expression to its corresponding single write index. * * @param state the cluster state containing all the data to resolve to expression to a concrete index - * @param request The request that defines how the an alias or an index need to be resolved to a concrete index + * @param request The request that defines how an alias or an index need to be resolved to a concrete index * and the expression that can be resolved to an alias or an index name. * @throws IllegalArgumentException if the index resolution does not lead to an index, or leads to more than one index * @return the write index obtained as a result of the index resolution @@ -759,7 +768,7 @@ public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) /** * Resolve an array of expressions to the set of indices and aliases that these expressions match. */ - public Set resolveExpressions(ClusterState state, String... expressions) { + public Set resolveExpressions(ClusterState state, String... expressions) { return resolveExpressions(state, IndicesOptions.lenientExpandOpen(), false, expressions); } @@ -768,7 +777,7 @@ public Set resolveExpressions(ClusterState state, String... expressions) * If {@param preserveDataStreams} is {@code true}, datastreams that are covered by the wildcards from the * {@param expressions} are returned as-is, without expanding them further to their respective backing indices. */ - public Set resolveExpressions( + public Set resolveExpressions( ClusterState state, IndicesOptions indicesOptions, boolean preserveDataStreams, @@ -786,10 +795,10 @@ public Set resolveExpressions( getNetNewSystemIndexPredicate() ); // unmodifiable without creating a new collection as it might contain many items - Collection resolved = resolveExpressionsToResources(context, expressions); - if (resolved instanceof Set) { + Collection resolved = resolveExpressionsToResources(context, expressions); + if (resolved instanceof Set) { // unmodifiable without creating a new collection as it might contain many items - return Collections.unmodifiableSet((Set) resolved); + return Collections.unmodifiableSet((Set) resolved); } else { return Set.copyOf(resolved); } @@ -802,7 +811,7 @@ public Set resolveExpressions( * the index itself - null is returned. Returns {@code null} if no filtering is required. * NOTE: The provided expressions must have been resolved already via {@link #resolveExpressionsToResources(Context, String...)}. */ - public String[] filteringAliases(ClusterState state, String index, Set resolvedExpressions) { + public String[] filteringAliases(ClusterState state, String index, Set resolvedExpressions) { return indexAliases(state, index, AliasMetadata::filteringRequired, DataStreamAlias::filteringRequired, false, resolvedExpressions); } @@ -829,26 +838,25 @@ public String[] indexAliases( Predicate requiredAlias, Predicate requiredDataStreamAlias, boolean skipIdentity, - Set resolvedExpressions + Set resolvedExpressions ) { - if (isAllIndices(resolvedExpressions)) { + if (isAllIndicesExpression(resolvedExpressions)) { return null; } - final IndexMetadata indexMetadata = state.metadata().getIndices().get(index); if (indexMetadata == null) { // Shouldn't happen throw new IndexNotFoundException(index); } - if (skipIdentity == false && resolvedExpressions.contains(index)) { + if (skipIdentity == false && resolvedExpressions.contains(new ResolvedExpression(index))) { return null; } IndexAbstraction ia = state.metadata().getIndicesLookup().get(index); DataStream dataStream = ia.getParentDataStream(); if (dataStream != null) { - if (skipIdentity == false && resolvedExpressions.contains(dataStream.getName())) { + if (skipIdentity == false && resolvedExpressions.contains(new ResolvedExpression(dataStream.getName()))) { // skip the filters when the request targets the data stream name return null; } @@ -857,11 +865,12 @@ public String[] indexAliases( if (iterateIndexAliases(dataStreamAliases.size(), resolvedExpressions.size())) { aliasesForDataStream = dataStreamAliases.values() .stream() - .filter(dataStreamAlias -> resolvedExpressions.contains(dataStreamAlias.getName())) + .filter(dataStreamAlias -> resolvedExpressions.contains(new ResolvedExpression(dataStreamAlias.getName()))) .filter(dataStreamAlias -> dataStreamAlias.getDataStreams().contains(dataStream.getName())) .toList(); } else { aliasesForDataStream = resolvedExpressions.stream() + .map(ResolvedExpression::resource) .map(dataStreamAliases::get) .filter(dataStreamAlias -> dataStreamAlias != null && dataStreamAlias.getDataStreams().contains(dataStream.getName())) .toList(); @@ -890,11 +899,12 @@ public String[] indexAliases( // faster to iterate indexAliases aliasCandidates = indexAliases.values() .stream() - .filter(aliasMetadata -> resolvedExpressions.contains(aliasMetadata.alias())) + .filter(aliasMetadata -> resolvedExpressions.contains(new ResolvedExpression(aliasMetadata.alias()))) .toArray(AliasMetadata[]::new); } else { // faster to iterate resolvedExpressions aliasCandidates = resolvedExpressions.stream() + .map(ResolvedExpression::resource) .map(indexAliases::get) .filter(Objects::nonNull) .toArray(AliasMetadata[]::new); @@ -937,12 +947,7 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab getSystemIndexAccessPredicate(), getNetNewSystemIndexPredicate() ); - final Collection resolvedExpressions = resolveExpressionsToResources(context, expressions); - - // TODO: it appears that this can never be true? - if (isAllIndices(resolvedExpressions)) { - return resolveSearchRoutingAllIndices(state.metadata(), routing); - } + final Collection resolvedExpressions = resolveExpressionsToResources(context, expressions); Map> routings = null; Set paramRouting = null; @@ -952,8 +957,8 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab paramRouting = Sets.newHashSet(Strings.splitStringByCommaToArray(routing)); } - for (String expression : resolvedExpressions) { - IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(expression); + for (ResolvedExpression resolvedExpression : resolvedExpressions) { + IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(resolvedExpression.resource()); if (indexAbstraction != null && indexAbstraction.getType() == Type.ALIAS) { for (int i = 0, n = indexAbstraction.getIndices().size(); i < n; i++) { Index index = indexAbstraction.getIndices().get(i); @@ -993,7 +998,7 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab } } else { // Index - routings = collectRoutings(routings, paramRouting, norouting, expression); + routings = collectRoutings(routings, paramRouting, norouting, resolvedExpression.resource()); } } @@ -1039,6 +1044,30 @@ public static Map> resolveSearchRoutingAllIndices(Metadata m return null; } + /** + * Identifies whether the array containing index names given as argument refers to all indices + * The empty or null array identifies all indices + * + * @param aliasesOrIndices the array containing index names + * @return true if the provided array maps to all indices, false otherwise + */ + public static boolean isAllIndicesExpression(Collection aliasesOrIndices) { + return aliasesOrIndices == null || aliasesOrIndices.isEmpty() || isExplicitAllPatternExpression(aliasesOrIndices); + } + + /** + * Identifies whether the array containing index names given as argument explicitly refers to all indices + * The empty or null array doesn't explicitly map to all indices + * + * @param aliasesOrIndices the array containing index names + * @return true if the provided array explicitly maps to all indices, false otherwise + */ + static boolean isExplicitAllPatternExpression(Collection aliasesOrIndices) { + return aliasesOrIndices != null + && aliasesOrIndices.size() == 1 + && Metadata.ALL.equals(aliasesOrIndices.iterator().next().resource()); + } + /** * Identifies whether the array containing index names given as argument refers to all indices * The empty or null array identifies all indices @@ -1334,14 +1363,14 @@ private WildcardExpressionResolver() { * Returns all the indices, data streams, and aliases, considering the open/closed, system, and hidden context parameters. * Depending on the context, returns the names of the data streams themselves or their backing indices. */ - public static Collection resolveAll(Context context) { - List concreteIndices = resolveEmptyOrTrivialWildcard(context); + public static Collection resolveAll(Context context) { + List concreteIndices = resolveEmptyOrTrivialWildcard(context); if (context.includeDataStreams() == false && context.getOptions().ignoreAliases()) { return concreteIndices; } - Set resolved = new HashSet<>(concreteIndices.size()); + Set resolved = new HashSet<>(concreteIndices.size()); context.getState() .metadata() .getIndicesLookup() @@ -1386,10 +1415,10 @@ private static IndexMetadata.State excludeState(IndicesOptions options) { * The {@param context} provides the current time-snapshot view of cluster state, as well as conditions * on whether to consider alias, data stream, system, and hidden resources. */ - static Set matchWildcardToResources(Context context, String wildcardExpression) { + static Set matchWildcardToResources(Context context, String wildcardExpression) { assert isWildcard(wildcardExpression); final SortedMap indicesLookup = context.getState().getMetadata().getIndicesLookup(); - Set matchedResources = new HashSet<>(); + Set matchedResources = new HashSet<>(); // this applies an initial pre-filtering in the case where the expression is a common suffix wildcard, eg "test*" if (Regex.isSuffixMatchPattern(wildcardExpression)) { for (IndexAbstraction ia : filterIndicesLookupForSuffixWildcard(indicesLookup, wildcardExpression).values()) { @@ -1416,7 +1445,7 @@ private static void maybeAddToResult( Context context, String wildcardExpression, IndexAbstraction indexAbstraction, - Set matchedResources + Set matchedResources ) { if (shouldExpandToIndexAbstraction(context, wildcardExpression, indexAbstraction)) { matchedResources.addAll(expandToOpenClosed(context, indexAbstraction)); @@ -1475,20 +1504,20 @@ private static Map filterIndicesLookupForSuffixWildcar * Data streams and aliases are interpreted to refer to multiple indices, * then all index resources are filtered by their open/closed status. */ - private static Set expandToOpenClosed(Context context, IndexAbstraction indexAbstraction) { + private static Set expandToOpenClosed(Context context, IndexAbstraction indexAbstraction) { final IndexMetadata.State excludeState = excludeState(context.getOptions()); - Set resources = new HashSet<>(); + Set resources = new HashSet<>(); if (context.isPreserveAliases() && indexAbstraction.getType() == Type.ALIAS) { - resources.add(indexAbstraction.getName()); + resources.add(new ResolvedExpression(indexAbstraction.getName())); } else if (context.isPreserveDataStreams() && indexAbstraction.getType() == Type.DATA_STREAM) { - resources.add(indexAbstraction.getName()); + resources.add(new ResolvedExpression(indexAbstraction.getName())); } else { if (shouldIncludeRegularIndices(context.getOptions())) { for (int i = 0, n = indexAbstraction.getIndices().size(); i < n; i++) { Index index = indexAbstraction.getIndices().get(i); IndexMetadata indexMetadata = context.state.metadata().index(index); if (indexMetadata.getState() != excludeState) { - resources.add(index.getName()); + resources.add(new ResolvedExpression(index.getName())); } } } @@ -1498,7 +1527,7 @@ private static Set expandToOpenClosed(Context context, IndexAbstraction Index index = dataStream.getFailureIndices().getIndices().get(i); IndexMetadata indexMetadata = context.state.metadata().index(index); if (indexMetadata.getState() != excludeState) { - resources.add(index.getName()); + resources.add(new ResolvedExpression(index.getName())); } } } @@ -1506,20 +1535,27 @@ private static Set expandToOpenClosed(Context context, IndexAbstraction return resources; } - private static List resolveEmptyOrTrivialWildcard(Context context) { + private static List resolveEmptyOrTrivialWildcard(Context context) { final String[] allIndices = resolveEmptyOrTrivialWildcardToAllIndices(context.getOptions(), context.getState().metadata()); if (context.systemIndexAccessLevel == SystemIndexAccessLevel.ALL) { - return List.of(allIndices); + List result = new ArrayList<>(allIndices.length); + for (int i = 0; i < allIndices.length; i++) { + result.add(new ResolvedExpression(allIndices[i])); + } + return result; } else { return resolveEmptyOrTrivialWildcardWithAllowedSystemIndices(context, allIndices); } } - private static List resolveEmptyOrTrivialWildcardWithAllowedSystemIndices(Context context, String[] allIndices) { - List filteredIndices = new ArrayList<>(allIndices.length); + private static List resolveEmptyOrTrivialWildcardWithAllowedSystemIndices( + Context context, + String[] allIndices + ) { + List filteredIndices = new ArrayList<>(allIndices.length); for (int i = 0; i < allIndices.length; i++) { if (shouldIncludeIndexAbstraction(context, allIndices[i])) { - filteredIndices.add(allIndices[i]); + filteredIndices.add(new ResolvedExpression(allIndices[i])); } } return filteredIndices; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 27d832241bfed..818bf2036e3b1 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -37,6 +37,7 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RecoverySource; @@ -1711,7 +1712,7 @@ interface IndexDeletionAllowedPredicate { IndexSettings indexSettings) -> canDeleteIndexContents(index); private final IndexDeletionAllowedPredicate ALWAYS_TRUE = (Index index, IndexSettings indexSettings) -> true; - public AliasFilter buildAliasFilter(ClusterState state, String index, Set resolvedExpressions) { + public AliasFilter buildAliasFilter(ClusterState state, String index, Set resolvedExpressions) { /* Being static, parseAliasFilter doesn't have access to whatever guts it needs to parse a query. Instead of passing in a bunch * of dependencies we pass in a function that can perform the parsing. */ CheckedFunction filterParser = bytes -> { diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index e17709ed78318..cec1fa8712ec1 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedSupplier; @@ -1610,7 +1611,7 @@ public boolean isForceExecution() { } } - public AliasFilter buildAliasFilter(ClusterState state, String index, Set resolvedExpressions) { + public AliasFilter buildAliasFilter(ClusterState state, String index, Set resolvedExpressions) { return indicesService.buildAliasFilter(state, index, resolvedExpressions); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java index 423de4b43088f..37957df46a796 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -229,9 +230,19 @@ public void testResolveHiddenProperlyWithDateMath() { .metadata(buildMetadata(new Object[][] {}, indices)) .build(); String[] requestedIndex = new String[] { "" }; - Set resolvedIndices = resolver.resolveExpressions(clusterState, IndicesOptions.LENIENT_EXPAND_OPEN, true, requestedIndex); + Set resolvedIndices = resolver.resolveExpressions( + clusterState, + IndicesOptions.LENIENT_EXPAND_OPEN, + true, + requestedIndex + ); assertThat(resolvedIndices.size(), is(1)); - assertThat(resolvedIndices, contains(oneOf("logs-pgsql-prod-" + todaySuffix, "logs-pgsql-prod-" + tomorrowSuffix))); + assertThat( + resolvedIndices, + contains( + oneOf(new ResolvedExpression("logs-pgsql-prod-" + todaySuffix), new ResolvedExpression("logs-pgsql-prod-" + tomorrowSuffix)) + ) + ); } public void testSystemIndexAccess() { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index 30895767c33c2..c2000ede8113b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata.State; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -55,6 +56,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex; @@ -1585,13 +1587,16 @@ public void testResolveExpressions() { .put(indexBuilder("test-1").state(State.OPEN).putAlias(AliasMetadata.builder("alias-1"))); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); - assertEquals(Set.of("alias-0", "alias-1"), indexNameExpressionResolver.resolveExpressions(state, "alias-*")); - assertEquals(Set.of("test-0", "alias-0", "alias-1"), indexNameExpressionResolver.resolveExpressions(state, "test-0", "alias-*")); + assertEquals(resolvedExpressionsSet("alias-0", "alias-1"), indexNameExpressionResolver.resolveExpressions(state, "alias-*")); assertEquals( - Set.of("test-0", "test-1", "alias-0", "alias-1"), + resolvedExpressionsSet("test-0", "alias-0", "alias-1"), + indexNameExpressionResolver.resolveExpressions(state, "test-0", "alias-*") + ); + assertEquals( + resolvedExpressionsSet("test-0", "test-1", "alias-0", "alias-1"), indexNameExpressionResolver.resolveExpressions(state, "test-*", "alias-*") ); - assertEquals(Set.of("test-1", "alias-1"), indexNameExpressionResolver.resolveExpressions(state, "*-1")); + assertEquals(resolvedExpressionsSet("test-1", "alias-1"), indexNameExpressionResolver.resolveExpressions(state, "*-1")); } public void testFilteringAliases() { @@ -1600,16 +1605,25 @@ public void testFilteringAliases() { .put(indexBuilder("test-1").state(State.OPEN).putAlias(AliasMetadata.builder("alias-1"))); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); - Set resolvedExpressions = Set.of("alias-0", "alias-1"); + Set resolvedExpressions = resolvedExpressionsSet("alias-0", "alias-1"); String[] strings = indexNameExpressionResolver.filteringAliases(state, "test-0", resolvedExpressions); assertArrayEquals(new String[] { "alias-0" }, strings); // concrete index supersedes filtering alias - resolvedExpressions = Set.of("test-0", "alias-0", "alias-1"); + resolvedExpressions = Set.of( + new ResolvedExpression("test-0"), + new ResolvedExpression("alias-0"), + new ResolvedExpression("alias-1") + ); strings = indexNameExpressionResolver.filteringAliases(state, "test-0", resolvedExpressions); assertNull(strings); - resolvedExpressions = Set.of("test-0", "test-1", "alias-0", "alias-1"); + resolvedExpressions = Set.of( + new ResolvedExpression("test-0"), + new ResolvedExpression("test-1"), + new ResolvedExpression("alias-0"), + new ResolvedExpression("alias-1") + ); strings = indexNameExpressionResolver.filteringAliases(state, "test-0", resolvedExpressions); assertNull(strings); } @@ -1623,7 +1637,7 @@ public void testIndexAliases() { .putAlias(AliasMetadata.builder("test-alias-non-filtering")) ); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "test-*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "test-*"); String[] strings = indexNameExpressionResolver.indexAliases(state, "test-0", x -> true, x -> true, true, resolvedExpressions); Arrays.sort(strings); @@ -1658,28 +1672,28 @@ public void testIndexAliasesDataStreamAliases() { ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); { // Only resolve aliases with with that refer to dataStreamName1 - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases(state, index, x -> true, x -> true, true, resolvedExpressions); assertThat(result, arrayContainingInAnyOrder("logs_foo", "logs", "logs_bar")); } { // Only resolve aliases with with that refer to dataStreamName2 - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); String index = backingIndex2.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases(state, index, x -> true, x -> true, true, resolvedExpressions); assertThat(result, arrayContainingInAnyOrder("logs_baz", "logs_baz2")); } { // Null is returned, because skipping identity check and resolvedExpressions contains the backing index name - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); String index = backingIndex2.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases(state, index, x -> true, x -> true, false, resolvedExpressions); assertThat(result, nullValue()); } { // Null is returned, because the wildcard expands to a list of aliases containing an unfiltered alias for dataStreamName1 - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*"); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases( state, @@ -1693,7 +1707,7 @@ public void testIndexAliasesDataStreamAliases() { } { // Null is returned, because an unfiltered alias is targeting the same data stream - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "logs_bar", "logs"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "logs_bar", "logs"); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases( state, @@ -1707,7 +1721,7 @@ public void testIndexAliasesDataStreamAliases() { } { // The filtered alias is returned because although we target the data stream name, skipIdentity is true - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, dataStreamName1, "logs"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, dataStreamName1, "logs"); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases( state, @@ -1721,7 +1735,7 @@ public void testIndexAliasesDataStreamAliases() { } { // Null is returned because we target the data stream name and skipIdentity is false - Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, dataStreamName1, "logs"); + Set resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, dataStreamName1, "logs"); String index = backingIndex1.getIndex().getName(); String[] result = indexNameExpressionResolver.indexAliases( state, @@ -1744,13 +1758,13 @@ public void testIndexAliasesSkipIdentity() { ); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); - Set resolvedExpressions = Set.of("test-0", "test-alias"); + Set resolvedExpressions = resolvedExpressionsSet("test-0", "test-alias"); String[] aliases = indexNameExpressionResolver.indexAliases(state, "test-0", x -> true, x -> true, false, resolvedExpressions); assertNull(aliases); aliases = indexNameExpressionResolver.indexAliases(state, "test-0", x -> true, x -> true, true, resolvedExpressions); assertArrayEquals(new String[] { "test-alias" }, aliases); - resolvedExpressions = Collections.singleton("other-alias"); + resolvedExpressions = Collections.singleton(new ResolvedExpression("other-alias")); aliases = indexNameExpressionResolver.indexAliases(state, "test-0", x -> true, x -> true, false, resolvedExpressions); assertArrayEquals(new String[] { "other-alias" }, aliases); aliases = indexNameExpressionResolver.indexAliases(state, "test-0", x -> true, x -> true, true, resolvedExpressions); @@ -1771,7 +1785,7 @@ public void testConcreteWriteIndexSuccessful() { x -> true, x -> true, true, - Set.of("test-0", "test-alias") + resolvedExpressionsSet("test-0", "test-alias") ); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); @@ -1853,7 +1867,7 @@ public void testConcreteWriteIndexWithWildcardExpansion() { x -> true, x -> true, true, - Set.of("test-0", "test-1", "test-alias") + resolvedExpressionsSet("test-0", "test-1", "test-alias") ); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); @@ -1891,7 +1905,7 @@ public void testConcreteWriteIndexWithNoWriteIndexWithSingleIndex() { x -> true, x -> true, true, - Set.of("test-0", "test-alias") + resolvedExpressionsSet("test-0", "test-alias") ); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); @@ -1927,7 +1941,7 @@ public void testConcreteWriteIndexWithNoWriteIndexWithMultipleIndices() { x -> true, x -> true, true, - Set.of("test-0", "test-1", "test-alias") + Set.of(new ResolvedExpression("test-0"), new ResolvedExpression("test-1"), new ResolvedExpression("test-alias")) ); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); @@ -1968,7 +1982,7 @@ public void testAliasResolutionNotAllowingMultipleIndices() { x -> true, x -> true, true, - Set.of("test-0", "test-1", "test-alias") + resolvedExpressionsSet("test-0", "test-1", "test-alias") ); Arrays.sort(strings); assertArrayEquals(new String[] { "test-alias" }, strings); @@ -3260,7 +3274,7 @@ public void testDateMathMixedArray() { Predicates.never(), Predicates.never() ); - Collection result = IndexNameExpressionResolver.resolveExpressionsToResources( + Collection result = IndexNameExpressionResolver.resolveExpressionsToResources( context, "name1", "<.marvel-{now/d}>", @@ -3268,7 +3282,15 @@ public void testDateMathMixedArray() { "<.logstash-{now/M{uuuu.MM}}>" ); assertThat(result.size(), equalTo(4)); - assertThat(result, contains("name1", dataMathIndex1, "name2", dateMathIndex2)); + assertThat( + result, + contains( + new ResolvedExpression("name1"), + new ResolvedExpression(dataMathIndex1), + new ResolvedExpression("name2"), + new ResolvedExpression(dateMathIndex2) + ) + ); } public void testMathExpressionSupport() { @@ -3466,4 +3488,8 @@ private static IndicesOptions.WildcardOptions doNotExpandWildcards(boolean lenie .allowEmptyExpressions(lenient) .build(); } + + private Set resolvedExpressionsSet(String... expressions) { + return Arrays.stream(expressions).map(ResolvedExpression::new).collect(Collectors.toSet()); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java index 6a26e7948784c..fd90e1210ce85 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java @@ -13,14 +13,17 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata.State; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; import org.elasticsearch.test.ESTestCase; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Set; import java.util.function.Predicate; +import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex; import static org.elasticsearch.common.util.set.Sets.newHashSet; @@ -48,19 +51,19 @@ public void testConvertWildcardsJustIndicesTests() { ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "ku*")), - equalTo(newHashSet("kuku")) + equalTo(resolvedExpressionsSet("kuku")) ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*")), - equalTo(newHashSet("testXXX", "testXYY", "testYYY")) + equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*")), - equalTo(newHashSet("testXXX", "testXYY")) + equalTo(resolvedExpressionsSet("testXXX", "testXYY")) ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*")), - equalTo(newHashSet("testXXX", "testXYY", "testYYY", "kuku")) + equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY", "kuku")) ); } @@ -81,7 +84,7 @@ public void testConvertWildcardsOpenClosedIndicesTests() { ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*")), - equalTo(newHashSet("testXXX", "testXXY", "testXYY")) + equalTo(resolvedExpressionsSet("testXXX", "testXXY", "testXYY")) ); context = new IndexNameExpressionResolver.Context( state, @@ -90,7 +93,7 @@ public void testConvertWildcardsOpenClosedIndicesTests() { ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*")), - equalTo(newHashSet("testXYY")) + equalTo(resolvedExpressionsSet("testXYY")) ); context = new IndexNameExpressionResolver.Context( state, @@ -99,7 +102,7 @@ public void testConvertWildcardsOpenClosedIndicesTests() { ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "testX*")), - equalTo(newHashSet("testXXX", "testXXY")) + equalTo(resolvedExpressionsSet("testXXX", "testXXY")) ); } @@ -122,19 +125,19 @@ public void testMultipleWildcards() { ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*")), - equalTo(newHashSet("testXXX", "testXXY", "testXYY")) + equalTo(resolvedExpressionsSet("testXXX", "testXXY", "testXYY")) ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*X*Y")), - equalTo(newHashSet("testXXY", "testXYY")) + equalTo(resolvedExpressionsSet("testXXY", "testXYY")) ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "kuku*Y*")), - equalTo(newHashSet("kukuYYY")) + equalTo(resolvedExpressionsSet("kukuYYY")) ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "*Y*")), - equalTo(newHashSet("testXXY", "testXYY", "testYYY", "kukuYYY")) + equalTo(resolvedExpressionsSet("testXXY", "testXYY", "testYYY", "kukuYYY")) ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, "test*Y*X")).size(), @@ -160,7 +163,7 @@ public void testAll() { ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), - equalTo(newHashSet("testXXX", "testXYY", "testYYY")) + equalTo(resolvedExpressionsSet("testXXX", "testXYY", "testYYY")) ); } @@ -202,7 +205,7 @@ public void testAllAliases() { ); assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), - equalTo(newHashSet("index-visible-alias")) + equalTo(resolvedExpressionsSet("index-visible-alias")) ); } } @@ -245,7 +248,7 @@ public void testAllDataStreams() { assertThat( newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), - equalTo(newHashSet(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis))) + equalTo(resolvedExpressionsSet(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis))) ); } @@ -389,46 +392,53 @@ public void testResolveAliases() { ); { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "foo_a*" ); - assertThat(indices, containsInAnyOrder("foo_index", "bar_index")); + assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_index"), new ResolvedExpression("bar_index"))); } { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( skipAliasesLenientContext, "foo_a*" ); assertEquals(0, indices.size()); } { - Set indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( + Set indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( skipAliasesStrictContext, "foo_a*" ); assertThat(indices, empty()); } { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "foo*" ); - assertThat(indices, containsInAnyOrder("foo_foo", "foo_index", "bar_index")); + assertThat( + indices, + containsInAnyOrder( + new ResolvedExpression("foo_foo"), + new ResolvedExpression("foo_index"), + new ResolvedExpression("bar_index") + ) + ); } { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( skipAliasesLenientContext, "foo*" ); - assertThat(indices, containsInAnyOrder("foo_foo", "foo_index")); + assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_foo"), new ResolvedExpression("foo_index"))); } { - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( skipAliasesStrictContext, "foo*" ); - assertThat(indices, containsInAnyOrder("foo_foo", "foo_index")); + assertThat(indices, containsInAnyOrder(new ResolvedExpression("foo_foo"), new ResolvedExpression("foo_index"))); } } @@ -472,15 +482,22 @@ public void testResolveDataStreams() { ); // data streams are not included but expression matches the data stream - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "foo_*" ); - assertThat(indices, containsInAnyOrder("foo_index", "foo_foo", "bar_index")); + assertThat( + indices, + containsInAnyOrder( + new ResolvedExpression("foo_index"), + new ResolvedExpression("foo_foo"), + new ResolvedExpression("bar_index") + ) + ); // data streams are not included and expression doesn't match the data steram indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(indicesAndAliasesContext, "bar_*"); - assertThat(indices, containsInAnyOrder("bar_bar", "bar_index")); + assertThat(indices, containsInAnyOrder(new ResolvedExpression("bar_bar"), new ResolvedExpression("bar_index"))); } { @@ -506,18 +523,18 @@ public void testResolveDataStreams() { ); // data stream's corresponding backing indices are resolved - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAliasesAndDataStreamsContext, "foo_*" ); assertThat( indices, containsInAnyOrder( - "foo_index", - "bar_index", - "foo_foo", - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis) + new ResolvedExpression("foo_index"), + new ResolvedExpression("bar_index"), + new ResolvedExpression("foo_foo"), + new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis)), + new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis)) ) ); @@ -529,12 +546,12 @@ public void testResolveDataStreams() { assertThat( indices, containsInAnyOrder( - "foo_index", - "bar_index", - "foo_foo", - "bar_bar", - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis) + new ResolvedExpression("foo_index"), + new ResolvedExpression("bar_index"), + new ResolvedExpression("foo_foo"), + new ResolvedExpression("bar_bar"), + new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis)), + new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis)) ) ); } @@ -563,18 +580,18 @@ public void testResolveDataStreams() { ); // data stream's corresponding backing indices are resolved - Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( + Collection indices = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAliasesDataStreamsAndHiddenIndices, "foo_*" ); assertThat( indices, containsInAnyOrder( - "foo_index", - "bar_index", - "foo_foo", - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis) + new ResolvedExpression("foo_index"), + new ResolvedExpression("bar_index"), + new ResolvedExpression("foo_foo"), + new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis)), + new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis)) ) ); @@ -586,12 +603,12 @@ public void testResolveDataStreams() { assertThat( indices, containsInAnyOrder( - "foo_index", - "bar_index", - "foo_foo", - "bar_bar", - DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis), - DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis) + new ResolvedExpression("foo_index"), + new ResolvedExpression("bar_index"), + new ResolvedExpression("foo_foo"), + new ResolvedExpression("bar_bar"), + new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis)), + new ResolvedExpression(DataStream.getDefaultBackingIndexName("foo_logs", 2, epochMillis)) ) ); } @@ -623,17 +640,36 @@ public void testMatchesConcreteIndicesWildcardAndAliases() { SystemIndexAccessLevel.NONE ); - Collection matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( + Collection matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources( indicesAndAliasesContext, "*" ); - assertThat(matches, containsInAnyOrder("bar_bar", "foo_foo", "foo_index", "bar_index")); + assertThat( + matches, + containsInAnyOrder( + new ResolvedExpression("bar_bar"), + new ResolvedExpression("foo_foo"), + new ResolvedExpression("foo_index"), + new ResolvedExpression("bar_index") + ) + ); matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(onlyIndicesContext, "*"); - assertThat(matches, containsInAnyOrder("bar_bar", "foo_foo", "foo_index", "bar_index")); + assertThat( + matches, + containsInAnyOrder( + new ResolvedExpression("bar_bar"), + new ResolvedExpression("foo_foo"), + new ResolvedExpression("foo_index"), + new ResolvedExpression("bar_index") + ) + ); matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(indicesAndAliasesContext, "foo*"); - assertThat(matches, containsInAnyOrder("foo_foo", "foo_index", "bar_index")); + assertThat( + matches, + containsInAnyOrder(new ResolvedExpression("foo_foo"), new ResolvedExpression("foo_index"), new ResolvedExpression("bar_index")) + ); matches = IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(onlyIndicesContext, "foo*"); - assertThat(matches, containsInAnyOrder("foo_foo", "foo_index")); + assertThat(matches, containsInAnyOrder(new ResolvedExpression("foo_foo"), new ResolvedExpression("foo_index"))); } private static IndexMetadata.Builder indexBuilder(String index, boolean hidden) { @@ -648,4 +684,8 @@ private static IndexMetadata.Builder indexBuilder(String index) { private static void assertWildcardResolvesToEmpty(IndexNameExpressionResolver.Context context, String wildcardExpression) { assertThat(IndexNameExpressionResolver.WildcardExpressionResolver.matchWildcardToResources(context, wildcardExpression), empty()); } + + private Set resolvedExpressionsSet(String... expressions) { + return Arrays.stream(expressions).map(ResolvedExpression::new).collect(Collectors.toSet()); + } } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index 36f7355a541c1..17975b7d18dd8 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexGraveyard; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -77,6 +78,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -677,27 +679,27 @@ public void testBuildAliasFilter() { ); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); { - AliasFilter result = indicesService.buildAliasFilter(state, "test-0", Set.of("test-alias-0")); + AliasFilter result = indicesService.buildAliasFilter(state, "test-0", resolvedExpressions("test-alias-0")); assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-0")); assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "bar"))); } { - AliasFilter result = indicesService.buildAliasFilter(state, "test-1", Set.of("test-alias-0")); + AliasFilter result = indicesService.buildAliasFilter(state, "test-1", resolvedExpressions("test-alias-0")); assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-0")); assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "bar"))); } { - AliasFilter result = indicesService.buildAliasFilter(state, "test-0", Set.of("test-alias-1")); + AliasFilter result = indicesService.buildAliasFilter(state, "test-0", resolvedExpressions("test-alias-1")); assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-1")); assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "baz"))); } { - AliasFilter result = indicesService.buildAliasFilter(state, "test-1", Set.of("test-alias-1")); + AliasFilter result = indicesService.buildAliasFilter(state, "test-1", resolvedExpressions("test-alias-1")); assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-1")); assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "bax"))); } { - AliasFilter result = indicesService.buildAliasFilter(state, "test-0", Set.of("test-alias-0", "test-alias-1")); + AliasFilter result = indicesService.buildAliasFilter(state, "test-0", resolvedExpressions("test-alias-0", "test-alias-1")); assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-0", "test-alias-1")); BoolQueryBuilder filter = (BoolQueryBuilder) result.getQueryBuilder(); assertThat(filter.filter(), empty()); @@ -706,7 +708,7 @@ public void testBuildAliasFilter() { assertThat(filter.should(), containsInAnyOrder(QueryBuilders.termQuery("foo", "baz"), QueryBuilders.termQuery("foo", "bar"))); } { - AliasFilter result = indicesService.buildAliasFilter(state, "test-1", Set.of("test-alias-0", "test-alias-1")); + AliasFilter result = indicesService.buildAliasFilter(state, "test-1", resolvedExpressions("test-alias-0", "test-alias-1")); assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-0", "test-alias-1")); BoolQueryBuilder filter = (BoolQueryBuilder) result.getQueryBuilder(); assertThat(filter.filter(), empty()); @@ -718,7 +720,7 @@ public void testBuildAliasFilter() { AliasFilter result = indicesService.buildAliasFilter( state, "test-0", - Set.of("test-alias-0", "test-alias-1", "test-alias-non-filtering") + resolvedExpressions("test-alias-0", "test-alias-1", "test-alias-non-filtering") ); assertThat(result.getAliases(), emptyArray()); assertThat(result.getQueryBuilder(), nullValue()); @@ -727,7 +729,7 @@ public void testBuildAliasFilter() { AliasFilter result = indicesService.buildAliasFilter( state, "test-1", - Set.of("test-alias-0", "test-alias-1", "test-alias-non-filtering") + resolvedExpressions("test-alias-0", "test-alias-1", "test-alias-non-filtering") ); assertThat(result.getAliases(), emptyArray()); assertThat(result.getQueryBuilder(), nullValue()); @@ -754,19 +756,19 @@ public void testBuildAliasFilterDataStreamAliases() { ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); { String index = backingIndex1.getIndex().getName(); - AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo")); + AliasFilter result = indicesService.buildAliasFilter(state, index, resolvedExpressions("logs_foo")); assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo")); assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "bar"))); } { String index = backingIndex2.getIndex().getName(); - AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo")); + AliasFilter result = indicesService.buildAliasFilter(state, index, resolvedExpressions("logs_foo")); assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo")); assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "baz"))); } { String index = backingIndex1.getIndex().getName(); - AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo", "logs")); + AliasFilter result = indicesService.buildAliasFilter(state, index, resolvedExpressions("logs_foo", "logs")); assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo", "logs")); BoolQueryBuilder filter = (BoolQueryBuilder) result.getQueryBuilder(); assertThat(filter.filter(), empty()); @@ -776,7 +778,7 @@ public void testBuildAliasFilterDataStreamAliases() { } { String index = backingIndex2.getIndex().getName(); - AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo", "logs")); + AliasFilter result = indicesService.buildAliasFilter(state, index, resolvedExpressions("logs_foo", "logs")); assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo", "logs")); BoolQueryBuilder filter = (BoolQueryBuilder) result.getQueryBuilder(); assertThat(filter.filter(), empty()); @@ -787,13 +789,13 @@ public void testBuildAliasFilterDataStreamAliases() { { // querying an unfiltered and a filtered alias for the same data stream should drop the filters String index = backingIndex1.getIndex().getName(); - AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo", "logs", "logs_bar")); + AliasFilter result = indicesService.buildAliasFilter(state, index, resolvedExpressions("logs_foo", "logs", "logs_bar")); assertThat(result, is(AliasFilter.EMPTY)); } { // similarly, querying the data stream name and a filtered alias should drop the filter String index = backingIndex1.getIndex().getName(); - AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs", dataStreamName1)); + AliasFilter result = indicesService.buildAliasFilter(state, index, resolvedExpressions("logs", dataStreamName1)); assertThat(result, is(AliasFilter.EMPTY)); } } @@ -846,4 +848,8 @@ public void testWithTempIndexServiceHandlesExistingIndex() throws Exception { return null; }); } + + private Set resolvedExpressions(String... expressions) { + return Arrays.stream(expressions).map(ResolvedExpression::new).collect(Collectors.toSet()); + } } From 23815590f89297c93304376dbb19478d71fe1ca9 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 10 Dec 2024 08:54:41 +0100 Subject: [PATCH 149/204] Update fallback setting (#118237) Update synthetic_source_fallback_to_stored_source setting to be an operator only setting. --- .../xpack/logsdb/SyntheticSourceLicenseService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java index e629f9b3998bb..f7f228859fb6d 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java @@ -38,7 +38,7 @@ final class SyntheticSourceLicenseService { "xpack.mapping.synthetic_source_fallback_to_stored_source", false, Setting.Property.NodeScope, - Setting.Property.Dynamic + Setting.Property.OperatorDynamic ); static final LicensedFeature.Momentary SYNTHETIC_SOURCE_FEATURE = LicensedFeature.momentary( From 5997fc3d6f51b69cc176ad45ee35fdc68a15b2b0 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Tue, 10 Dec 2024 10:12:30 +0200 Subject: [PATCH 150/204] Introduce data stream options via dedicated classes (#117357) In this PR we move the failure store configuration from the `data_stream` in a composable index template to the `template` section under the `data_stream_options`. The reason for this change is that we want to be able to compose the failure store configuration via component templates. **Previous composable index template** ``` { "index_patterns": [....], "template": { ...... }, "data_stream": { "failure_store": true } } ``` **New composable index template** ``` { "index_patterns": [....], "template": { "data_stream_options": { "failure_store": { "enabled": true } }, ...... }, "data_stream": {} } ``` **Composition logic** | Component Template A | Composable Index Template | Resolved data stream options | |-------------------------|-----------------------------|-------------------------------| | `{"failure_store": {"enabled": true}}` | - |`{"failure_store": {"enabled": true}}`| | `{"failure_store": {"enabled": true}}` | `{"failure_store": {"enabled": false}}` |`{"failure_store": {"enabled": false}}`| | - | `{"failure_store": {"enabled": true}}` |`{"failure_store": {"enabled": true}}`| | `{"failure_store": null}` | `{"failure_store": {"enabled": true}}` |`{"failure_store": {"enabled": true}}`| | `{"failure_store": {"enabled": true}}` | `{"failure_store": null}` |`{}`| More context and discussions can be found in the comments of https://github.com/elastic/elasticsearch/pull/113863. --- modules/data-streams/build.gradle | 28 +++ .../datastreams/DataStreamIT.java | 4 +- ...lureStoreMetricsWithIncrementalBulkIT.java | 7 +- .../IngestFailureStoreMetricsIT.java | 7 +- .../DataStreamLifecycleServiceIT.java | 4 +- .../datastreams/DataStreamOptionsIT.java | 59 ++++- .../datastreams/FailureStoreQueryParamIT.java | 6 +- .../test/data_stream/10_basic.yml | 43 +++- .../test/data_stream/150_tsdb.yml | 15 +- .../data_stream/170_modify_data_stream.yml | 15 +- .../190_failure_store_redirection.yml | 144 +++++------- .../200_rollover_failure_store.yml | 40 ++-- .../30_auto_create_data_stream.yml | 15 +- .../org/elasticsearch/TransportVersions.java | 1 + .../post/SimulateIndexTemplateResponse.java | 5 +- .../TransportSimulateIndexTemplateAction.java | 9 +- .../action/bulk/TransportBulkAction.java | 18 +- .../metadata/ComposableIndexTemplate.java | 42 ++-- .../cluster/metadata/DataStream.java | 3 +- .../metadata/DataStreamFailureStore.java | 91 +++++++- .../cluster/metadata/DataStreamOptions.java | 128 ++++++++++- .../MetadataCreateDataStreamService.java | 11 +- .../MetadataIndexTemplateService.java | 105 ++++++++- .../cluster/metadata/ResettableValue.java | 216 ++++++++++++++++++ .../cluster/metadata/Template.java | 70 +++++- .../RestPutComponentTemplateAction.java | 10 + .../RestPutComposableIndexTemplateAction.java | 10 + .../GetComponentTemplateResponseTests.java | 8 +- .../action/bulk/BulkOperationTests.java | 7 +- .../bulk/TransportBulkActionIngestTests.java | 4 +- .../action/bulk/TransportBulkActionTests.java | 39 +++- .../metadata/ComponentTemplateTests.java | 49 ++-- .../ComposableIndexTemplateTests.java | 6 +- .../DataStreamFailureStoreTemplateTests.java | 65 ++++++ .../metadata/DataStreamFailureStoreTests.java | 2 +- .../DataStreamOptionsTemplateTests.java | 116 ++++++++++ .../metadata/DataStreamTemplateTests.java | 11 +- .../cluster/metadata/DataStreamTests.java | 2 +- .../MetadataCreateDataStreamServiceTests.java | 18 +- .../MetadataIndexTemplateServiceTests.java | 141 +++++++++++- .../metadata/ResettableValueTests.java | 103 +++++++++ .../metadata/DataStreamTestHelper.java | 20 +- .../xpack/core/DataStreamRestIT.java | 14 +- .../rest-api-spec/test/20_failure_store.yml | 11 +- 44 files changed, 1458 insertions(+), 264 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/cluster/metadata/ResettableValue.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTemplateTests.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/ResettableValueTests.java diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index b017ae9921b0e..8ae56101ef01e 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -42,4 +42,32 @@ if (buildParams.isSnapshotBuild() == false) { tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("data_stream/10_basic/Create hidden data stream", "warning does not exist for compatibility") + + // Failure store configuration changed on 8.18 (earlier versions behind feature flag) + task.skipTest("data_stream/10_basic/Create data stream with failure store", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/10_basic/Delete data stream with failure store", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/10_basic/Delete data stream with failure store uninitialized", "Configuring the failure store via data stream templates is not supported anymore.") + + task.skipTest("data_stream/30_auto_create_data_stream/Don't initialize failure store during data stream auto-creation on successful index", "Configuring the failure store via data stream templates is not supported anymore.") + + task.skipTest("data_stream/150_tsdb/TSDB failures go to failure store", "Configuring the failure store via data stream templates is not supported anymore.") + + task.skipTest("data_stream/170_modify_data_stream/Modify a data stream's failure store", "Configuring the failure store via data stream templates is not supported anymore.") + + task.skipTest("data_stream/190_failure_store_redirection/Failure redirects to original failure store during index change if final pipeline changes target", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Ensure failure is redirected to correct failure store after a reroute processor", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Test failure store status with bulk request", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Redirect ingest failure in data stream to failure store", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Failure redirects to correct failure store when pipeline loop is detected", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Failure redirects to correct failure store when index loop is detected", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Failure redirects to original failure store during index change if self referenced", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Redirect shard failure in data stream to failure store", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Version conflicts are not redirected to failure store", "Configuring the failure store via data stream templates is not supported anymore.") + + task.skipTest("data_stream/200_rollover_failure_store/Lazily roll over a data stream's failure store after an ingest failure", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/200_rollover_failure_store/A failure store marked for lazy rollover should only be rolled over when there is a failure", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/200_rollover_failure_store/Roll over a data stream's failure store without conditions", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/200_rollover_failure_store/Lazily roll over a data stream's failure store after a shard failure", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/200_rollover_failure_store/Don't roll over a data stream's failure store when conditions aren't met", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/200_rollover_failure_store/Roll over a data stream's failure store with conditions", "Configuring the failure store via data stream templates is not supported anymore.") }) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 777ddc28fefdc..0e03045a090f8 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -67,6 +67,7 @@ import org.elasticsearch.cluster.metadata.DataStreamAction; import org.elasticsearch.cluster.metadata.DataStreamAlias; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexMetadataStats; import org.elasticsearch.cluster.metadata.IndexWriteLoad; @@ -2447,9 +2448,10 @@ static void putComposableIndexTemplate( .mappings(mappings == null ? null : CompressedXContent.fromJSON(mappings)) .aliases(aliases) .lifecycle(lifecycle) + .dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(withFailureStore)) ) .metadata(metadata) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, withFailureStore)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) .build() ); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/FailureStoreMetricsWithIncrementalBulkIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/FailureStoreMetricsWithIncrementalBulkIT.java index 2c9b7417b2832..d27ec04179e1a 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/FailureStoreMetricsWithIncrementalBulkIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/FailureStoreMetricsWithIncrementalBulkIT.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -165,8 +166,8 @@ private void createDataStreamWithFailureStore() throws IOException { request.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(List.of(DATA_STREAM_NAME + "*")) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, true)) - .template(new Template(null, new CompressedXContent(""" + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(Template.builder().mappings(new CompressedXContent(""" { "dynamic": false, "properties": { @@ -177,7 +178,7 @@ private void createDataStreamWithFailureStore() throws IOException { "type": "long" } } - }"""), null)) + }""")).dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(true))) .build() ); assertAcked(safeGet(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request))); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java index 96def04069e24..e9eaf7b5faddb 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; @@ -283,8 +284,8 @@ private void putComposableIndexTemplate(boolean failureStore) throws IOException request.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(List.of(dataStream + "*")) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, failureStore)) - .template(new Template(null, new CompressedXContent(""" + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(Template.builder().mappings(new CompressedXContent(""" { "dynamic": false, "properties": { @@ -295,7 +296,7 @@ private void putComposableIndexTemplate(boolean failureStore) throws IOException "type": "long" } } - }"""), null)) + }""")).dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(failureStore))) .build() ); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java index 89c440f5edf8b..19067d85a6805 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java @@ -40,6 +40,7 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamAction; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -1226,9 +1227,10 @@ static void putComposableIndexTemplate( .settings(settings) .mappings(mappings == null ? null : CompressedXContent.fromJSON(mappings)) .lifecycle(lifecycle) + .dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(withFailureStore)) ) .metadata(metadata) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, withFailureStore)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) .build() ); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java index 980cc32a12c68..de6b7a682324e 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java @@ -11,12 +11,14 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.junit.Before; import java.io.IOException; import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -40,10 +42,14 @@ public void setup() throws IOException { "template": { "settings": { "number_of_replicas": 0 + }, + "data_stream_options": { + "failure_store": { + "enabled": true + } } }, "data_stream": { - "failure_store": true } } """); @@ -59,12 +65,63 @@ public void setup() throws IOException { assertThat(dataStreams.size(), is(1)); Map dataStream = (Map) dataStreams.get(0); assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME)); + assertThat(((Map) dataStream.get("failure_store")).get("enabled"), is(true)); List backingIndices = getIndices(dataStream); assertThat(backingIndices.size(), is(1)); List failureStore = getFailureStore(dataStream); assertThat(failureStore.size(), is(1)); } + public void testExplicitlyResetDataStreamOptions() throws IOException { + Request putComponentTemplateRequest = new Request("POST", "/_component_template/with-options"); + putComponentTemplateRequest.setJsonEntity(""" + { + "template": { + "data_stream_options": { + "failure_store": { + "enabled": true + } + } + } + } + """); + assertOK(client().performRequest(putComponentTemplateRequest)); + + Request invalidRequest = new Request("POST", "/_index_template/other-template"); + invalidRequest.setJsonEntity(""" + { + "index_patterns": ["something-else"], + "composed_of" : ["with-options"], + "template": { + "settings": { + "number_of_replicas": 0 + } + } + } + """); + Exception error = expectThrows(ResponseException.class, () -> client().performRequest(invalidRequest)); + assertThat( + error.getMessage(), + containsString("specifies data stream options that can only be used in combination with a data stream") + ); + + // Check that when we nullify the data stream options we can create use any component template in a non data stream template + Request otherRequest = new Request("POST", "/_index_template/other-template"); + otherRequest.setJsonEntity(""" + { + "index_patterns": ["something-else"], + "composed_of" : ["with-options"], + "template": { + "settings": { + "number_of_replicas": 0 + }, + "data_stream_options": null + } + } + """); + assertOK(client().performRequest(otherRequest)); + } + public void testEnableDisableFailureStore() throws IOException { { assertAcknowledged(client().performRequest(new Request("DELETE", "/_data_stream/" + DATA_STREAM_NAME + "/_options"))); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java index 20ec26c0c9341..85b914be30b2c 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java @@ -42,10 +42,14 @@ public void setup() throws IOException { "template": { "settings": { "number_of_replicas": 0 + }, + "data_stream_options": { + "failure_store": { + "enabled": true + } } }, "data_stream": { - "failure_store": true } } """); diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml index 044ea90fec1af..0d19f555d10a4 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml @@ -212,8 +212,12 @@ setup: --- "Create data stream with failure store": - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores default settings changed in 8.15+" + test_runner_features: [ capabilities, allowed_warnings ] + reason: "Data stream failure stores config in templates was added in 8.18+" + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] - do: ingest.put_pipeline: @@ -256,8 +260,7 @@ setup: name: my-template4 body: index_patterns: [ failure-data-stream1, failure-data-stream2 ] - data_stream: - failure_store: true + data_stream: {} template: settings: index: @@ -269,6 +272,9 @@ setup: type: date count: type: long + data_stream_options: + failure_store: + enabled: true - do: indices.create_data_stream: @@ -632,8 +638,12 @@ setup: --- "Delete data stream with failure store": - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores REST structure changed in 8.15+" + reason: "Data stream failure stores config in templates was added in 8.18+" + test_runner_features: [ allowed_warnings, capabilities ] + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] - do: allowed_warnings: @@ -642,8 +652,7 @@ setup: name: my-template4 body: index_patterns: [ failure-data-stream1 ] - data_stream: - failure_store: true + data_stream: {} template: mappings: properties: @@ -651,6 +660,9 @@ setup: type: date count: type: long + data_stream_options: + failure_store: + enabled: true - do: indices.create_data_stream: @@ -722,8 +734,12 @@ setup: --- "Delete data stream with failure store uninitialized": - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores REST structure changed in 8.15+" + reason: "Data stream failure stores config in templates was added in 8.18+" + test_runner_features: [ capabilities, allowed_warnings ] + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] - do: allowed_warnings: @@ -732,8 +748,11 @@ setup: name: my-template4 body: index_patterns: [ failure-data-stream1 ] - data_stream: - failure_store: true + data_stream: {} + template: + data_stream_options: + failure_store: + enabled: true - do: indices.create_data_stream: diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 3fbf85ab1e702..9ea3bfefabdf8 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -185,9 +185,12 @@ index without timestamp: --- TSDB failures go to failure store: - requires: - cluster_features: ["data_stream.failure_store.tsdb_fix"] - reason: "tests tsdb failure store fixes in 8.16.0 that catch timestamp errors that happen earlier in the process and redirect them to the failure store." - + reason: "Data stream failure stores config in templates was added in 8.18+" + test_runner_features: [ capabilities, allowed_warnings ] + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] - do: allowed_warnings: - "index template [my-template2] has index patterns [fs-k8s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" @@ -195,8 +198,7 @@ TSDB failures go to failure store: name: my-template2 body: index_patterns: [ "fs-k8s*" ] - data_stream: - failure_store: true + data_stream: {} template: settings: index: @@ -207,6 +209,9 @@ TSDB failures go to failure store: time_series: start_time: 2021-04-28T00:00:00Z end_time: 2021-04-29T00:00:00Z + data_stream_options: + failure_store: + enabled: true mappings: properties: "@timestamp": diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml index 3c6d29d939226..13f79e95d99f4 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml @@ -92,9 +92,12 @@ --- "Modify a data stream's failure store": - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [ "allowed_warnings" ] + reason: "Data stream failure stores config in templates was added in 8.18+" + test_runner_features: [ capabilities, allowed_warnings ] + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] - do: allowed_warnings: @@ -103,8 +106,7 @@ name: my-template body: index_patterns: [data-*] - data_stream: - failure_store: true + data_stream: {} template: mappings: properties: @@ -112,6 +114,9 @@ type: date count: type: long + data_stream_options: + failure_store: + enabled: true - do: indices.create_data_stream: diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml index 9b5a9dae8bc0a..2f6b7a0bff34b 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml @@ -1,3 +1,15 @@ +setup: + - requires: + reason: "Data stream options was added in 8.18+" + test_runner_features: [ capabilities, allowed_warnings, contains ] + capabilities: + - method: POST + path: /{index}/_doc + capabilities: [ 'failure_store_status' ] + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] + --- teardown: - do: @@ -32,13 +44,6 @@ teardown: --- "Redirect ingest failure in data stream to failure store": - - requires: - reason: "Failure store status was added in 8.16+" - test_runner_features: [capabilities, allowed_warnings, contains] - capabilities: - - method: POST - path: /{index}/_doc - capabilities: [ 'failure_store_status' ] - do: ingest.put_pipeline: id: "failing_pipeline" @@ -78,14 +83,16 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: default_pipeline: "parent_failing_pipeline" + data_stream_options: + failure_store: + enabled: true - do: index: @@ -147,14 +154,6 @@ teardown: --- "Redirect shard failure in data stream to failure store": - - requires: - reason: "Failure store status was added in 8.16+" - test_runner_features: [ capabilities, allowed_warnings, contains ] - capabilities: - - method: POST - path: /{index}/_doc - capabilities: [ 'failure_store_status' ] - - do: allowed_warnings: - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation" @@ -162,8 +161,7 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 @@ -174,7 +172,9 @@ teardown: type: date count: type: long - + data_stream_options: + failure_store: + enabled: true - do: index: @@ -231,13 +231,6 @@ teardown: --- "Ensure failure is redirected to correct failure store after a reroute processor": - - requires: - test_runner_features: [allowed_warnings, capabilities] - reason: "Failure store status was added in 8.16+" - capabilities: - - method: POST - path: /{index}/_doc - capabilities: [ 'failure_store_status' ] - do: ingest.put_pipeline: id: "failing_pipeline" @@ -262,14 +255,16 @@ teardown: name: destination_template body: index_patterns: destination-data-stream - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: default_pipeline: "failing_pipeline" + data_stream_options: + failure_store: + enabled: true - do: indices.create_data_stream: @@ -331,11 +326,6 @@ teardown: --- "Failure redirects to original failure store during index change if self referenced": - - requires: - cluster_features: [ "gte_v8.15.0" ] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [ allowed_warnings, contains ] - - do: ingest.put_pipeline: id: "failing_pipeline" @@ -365,14 +355,16 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: default_pipeline: "failing_pipeline" + data_stream_options: + failure_store: + enabled: true - do: index: @@ -430,14 +422,6 @@ teardown: --- "Failure redirects to original failure store during index change if final pipeline changes target": - - requires: - reason: "Failure store status was added in 8.16+" - test_runner_features: [ capabilities, allowed_warnings, contains ] - capabilities: - - method: POST - path: /{index}/_doc - capabilities: [ 'failure_store_status' ] - - do: ingest.put_pipeline: id: "change_index_pipeline" @@ -462,14 +446,16 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: final_pipeline: "change_index_pipeline" + data_stream_options: + failure_store: + enabled: true - do: index: @@ -526,14 +512,6 @@ teardown: --- "Failure redirects to correct failure store when index loop is detected": - - requires: - reason: "Failure store status was added in 8.16+" - test_runner_features: [ capabilities, allowed_warnings, contains ] - capabilities: - - method: POST - path: /{index}/_doc - capabilities: [ 'failure_store_status' ] - - do: ingest.put_pipeline: id: "send_to_destination" @@ -575,14 +553,16 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: default_pipeline: "send_to_destination" + data_stream_options: + failure_store: + enabled: true - do: allowed_warnings: @@ -591,14 +571,16 @@ teardown: name: destination_logs_template body: index_patterns: destination-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: default_pipeline: "send_back_to_original" + data_stream_options: + failure_store: + enabled: true - do: index: @@ -657,14 +639,6 @@ teardown: --- "Failure redirects to correct failure store when pipeline loop is detected": - - requires: - reason: "Failure store status was added in 8.16+" - test_runner_features: [ capabilities, allowed_warnings, contains ] - capabilities: - - method: POST - path: /{index}/_doc - capabilities: [ 'failure_store_status' ] - - do: ingest.put_pipeline: id: "step_1" @@ -706,14 +680,16 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: default_pipeline: "step_1" + data_stream_options: + failure_store: + enabled: true - do: index: @@ -773,9 +749,6 @@ teardown: --- "Version conflicts are not redirected to failure store": - - requires: - test_runner_features: [ allowed_warnings] - - do: allowed_warnings: - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation" @@ -783,8 +756,7 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 @@ -795,6 +767,9 @@ teardown: type: date count: type: long + data_stream_options: + failure_store: + enabled: true - do: bulk: @@ -812,17 +787,6 @@ teardown: --- "Test failure store status with bulk request": - - requires: - test_runner_features: [ allowed_warnings, capabilities ] - reason: "Failure store status was added in 8.16+" - capabilities: - - method: POST - path: /_bulk - capabilities: [ 'failure_store_status' ] - - method: PUT - path: /_bulk - capabilities: [ 'failure_store_status' ] - - do: allowed_warnings: - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation" @@ -830,8 +794,7 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 @@ -842,6 +805,9 @@ teardown: type: date count: type: long + data_stream_options: + failure_store: + enabled: true - do: allowed_warnings: - "index template [no-fs] has index patterns [no-fs*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [no-fs] will take precedence during new index creation" @@ -849,8 +815,7 @@ teardown: name: no-fs body: index_patterns: no-fs* - data_stream: - failure_store: false + data_stream: {} template: settings: number_of_shards: 1 @@ -861,6 +826,9 @@ teardown: type: date count: type: long + data_stream_options: + failure_store: + enabled: false - do: diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml index 0742435f045fb..cc3a11ffde5e8 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml @@ -1,9 +1,15 @@ --- setup: - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [allowed_warnings, contains, capabilities] + reason: "Data stream failure stores config in templates was added in 8.16+" + test_runner_features: [ capabilities, allowed_warnings ] + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] + - method: POST + path: /{index}/_rollover + capabilities: [ 'lazy-rollover-failure-store' ] - do: allowed_warnings: @@ -12,8 +18,7 @@ setup: name: my-template body: index_patterns: [data-*] - data_stream: - failure_store: true + data_stream: {} template: mappings: properties: @@ -21,6 +26,9 @@ setup: type: date count: type: long + data_stream_options: + failure_store: + enabled: true - do: indices.create_data_stream: @@ -145,14 +153,6 @@ teardown: --- "Lazily roll over a data stream's failure store after a shard failure": - - requires: - reason: "data stream failure store lazy rollover only supported in 8.15+" - test_runner_features: [allowed_warnings, capabilities] - capabilities: - - method: POST - path: /{index}/_rollover - capabilities: [lazy-rollover-failure-store] - # Initialize failure store - do: index: @@ -215,14 +215,6 @@ teardown: --- "Lazily roll over a data stream's failure store after an ingest failure": - - requires: - reason: "data stream failure store lazy rollover only supported in 8.15+" - test_runner_features: [allowed_warnings, capabilities] - capabilities: - - method: POST - path: /{index}/_rollover - capabilities: [lazy-rollover-failure-store] - - do: ingest.put_pipeline: id: "failing_pipeline" @@ -246,12 +238,14 @@ teardown: name: my-template body: index_patterns: [data-*] - data_stream: - failure_store: true + data_stream: {} template: settings: index: default_pipeline: "failing_pipeline" + data_stream_options: + failure_store: + enabled: true - do: indices.create_data_stream: diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml index 61d17c3d675cf..60500767213af 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml @@ -50,9 +50,12 @@ --- "Don't initialize failure store during data stream auto-creation on successful index": - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: allowed_warnings + reason: "Data stream failure stores config in templates was added in 8.18+" + test_runner_features: [allowed_warnings, capabilities] + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] - do: allowed_warnings: @@ -61,12 +64,14 @@ name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 + data_stream_options: + failure_store: + enabled: true - do: index: diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 40a209c5f0f14..36a8e3c40862e 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -140,6 +140,7 @@ static TransportVersion def(int id) { public static final TransportVersion SOURCE_MODE_TELEMETRY = def(8_802_00_0); public static final TransportVersion NEW_REFRESH_CLUSTER_BLOCK = def(8_803_00_0); public static final TransportVersion RETRIES_AND_OPERATIONS_IN_BLOBSTORE_STATS = def(8_804_00_0); + public static final TransportVersion ADD_DATA_STREAM_OPTIONS_TO_TEMPLATES = def(8_805_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index 80e6fbfe051a4..a521dac60e96a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; +import org.elasticsearch.cluster.metadata.ResettableValue; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -111,9 +112,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (this.resolvedTemplate != null) { + if (resolvedTemplate != null) { builder.field(TEMPLATE.getPreferredName()); - this.resolvedTemplate.toXContent(builder, params, rolloverConfiguration); + resolvedTemplate.toXContent(builder, ResettableValue.hideResetValues(params), rolloverConfiguration); } if (this.overlappingTemplates != null) { builder.startArray(OVERLAPPING.getPreferredName()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 94d9b87467ea8..5f98852148ed4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -61,6 +61,7 @@ import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV1Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV2Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveDataStreamOptions; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveLifecycle; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings; @@ -348,7 +349,13 @@ public static Template resolveTemplate( if (template.getDataStreamTemplate() != null && lifecycle == null && isDslOnlyMode) { lifecycle = DataStreamLifecycle.DEFAULT; } - return new Template(settings, mergedMapping, aliasesByName, lifecycle); + return new Template( + settings, + mergedMapping, + aliasesByName, + lifecycle, + resolveDataStreamOptions(simulatedState.metadata(), matchingTemplate) + ); } private static IndexLongFieldRange getEventIngestedRange(String indexName, ClusterState simulatedState) { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index cef68324e2a45..e2c73349b93ec 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -35,6 +35,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamOptions; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; @@ -42,6 +43,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.core.Nullable; import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.VersionType; @@ -638,12 +640,16 @@ private static Boolean resolveFailureStoreFromMetadata(String indexName, Metadat } /** - * Determines if an index name is associated with an index template that has a data stream failure store enabled. + * Determines if an index name is associated with an index template that has a data stream failure store enabled. Since failure store is + * a data stream feature, the method returns true/false only if it is a data stream template, otherwise null. * @param indexName The index name to check. * @param metadata Cluster state metadata. - * @return true if the given index name corresponds to an index template with a data stream failure store enabled. + * @return true the associated index template has failure store enabled, false if the failure store is disabled or it's not specified, + * and null if the template is not a data stream template. + * Visible for testing */ - private static Boolean resolveFailureStoreFromTemplate(String indexName, Metadata metadata) { + @Nullable + static Boolean resolveFailureStoreFromTemplate(String indexName, Metadata metadata) { if (indexName == null) { return null; } @@ -656,7 +662,11 @@ private static Boolean resolveFailureStoreFromTemplate(String indexName, Metadat ComposableIndexTemplate composableIndexTemplate = metadata.templatesV2().get(template); if (composableIndexTemplate.getDataStreamTemplate() != null) { // Check if the data stream has the failure store enabled - return composableIndexTemplate.getDataStreamTemplate().hasFailureStore(); + DataStreamOptions dataStreamOptions = MetadataIndexTemplateService.resolveDataStreamOptions( + composableIndexTemplate, + metadata.componentTemplates() + ).mapAndGet(DataStreamOptions.Template::toDataStreamOptions); + return dataStreamOptions != null && dataStreamOptions.isFailureStoreEnabled(); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index ae7cff6312155..998217e93c426 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -372,16 +373,14 @@ public static class DataStreamTemplate implements Writeable, ToXContentObject { private static final ParseField HIDDEN = new ParseField("hidden"); private static final ParseField ALLOW_CUSTOM_ROUTING = new ParseField("allow_custom_routing"); + // Remove this after this PR gets backported + @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT) private static final ParseField FAILURE_STORE = new ParseField("failure_store"); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "data_stream_template", false, - args -> new DataStreamTemplate( - args[0] != null && (boolean) args[0], - args[1] != null && (boolean) args[1], - DataStream.isFailureStoreFeatureFlagEnabled() && args[2] != null && (boolean) args[2] - ) + args -> new DataStreamTemplate(args[0] != null && (boolean) args[0], args[1] != null && (boolean) args[1]) ); static { @@ -394,20 +393,14 @@ public static class DataStreamTemplate implements Writeable, ToXContentObject { private final boolean hidden; private final boolean allowCustomRouting; - private final boolean failureStore; public DataStreamTemplate() { - this(false, false, false); + this(false, false); } public DataStreamTemplate(boolean hidden, boolean allowCustomRouting) { - this(hidden, allowCustomRouting, false); - } - - public DataStreamTemplate(boolean hidden, boolean allowCustomRouting, boolean failureStore) { this.hidden = hidden; this.allowCustomRouting = allowCustomRouting; - this.failureStore = failureStore; } DataStreamTemplate(StreamInput in) throws IOException { @@ -425,10 +418,9 @@ public DataStreamTemplate(boolean hidden, boolean allowCustomRouting, boolean fa boolean value = in.readBoolean(); assert value == false : "expected false, because this used to be an optional enum that never got set"; } - if (in.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION)) { - failureStore = in.readBoolean(); - } else { - failureStore = false; + if (in.getTransportVersion() + .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, TransportVersions.ADD_DATA_STREAM_OPTIONS_TO_TEMPLATES)) { + in.readBoolean(); } } @@ -458,10 +450,6 @@ public boolean isAllowCustomRouting() { return allowCustomRouting; } - public boolean hasFailureStore() { - return failureStore; - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(hidden); @@ -472,8 +460,11 @@ public void writeTo(StreamOutput out) throws IOException { // See comment in constructor. out.writeBoolean(false); } - if (out.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION)) { - out.writeBoolean(failureStore); + if (out.getTransportVersion() + .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, TransportVersions.ADD_DATA_STREAM_OPTIONS_TO_TEMPLATES)) { + // Previous versions expect the failure store to be configured via the DataStreamTemplate. We add it here, so we don't break + // the serialisation, but we do not care to preserve the value because this feature is still behind a feature flag. + out.writeBoolean(false); } } @@ -482,9 +473,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field("hidden", hidden); builder.field(ALLOW_CUSTOM_ROUTING.getPreferredName(), allowCustomRouting); - if (DataStream.isFailureStoreFeatureFlagEnabled()) { - builder.field(FAILURE_STORE.getPreferredName(), failureStore); - } builder.endObject(); return builder; } @@ -494,12 +482,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; DataStreamTemplate that = (DataStreamTemplate) o; - return hidden == that.hidden && allowCustomRouting == that.allowCustomRouting && failureStore == that.failureStore; + return hidden == that.hidden && allowCustomRouting == that.allowCustomRouting; } @Override public int hashCode() { - return Objects.hash(hidden, allowCustomRouting, failureStore); + return Objects.hash(hidden, allowCustomRouting); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 979434950cf7a..1c6206a4815eb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -424,7 +424,7 @@ public boolean isAllowCustomRouting() { * @return true, if the user has explicitly enabled the failure store. */ public boolean isFailureStoreEnabled() { - return dataStreamOptions.failureStore() != null && dataStreamOptions.failureStore().isExplicitlyEnabled(); + return dataStreamOptions.isFailureStoreEnabled(); } @Nullable @@ -1188,6 +1188,7 @@ public void writeTo(StreamOutput out) throws IOException { ); // The fields behind the feature flag should always be last. if (DataStream.isFailureStoreFeatureFlagEnabled()) { + // Should be removed after backport PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), FAILURE_STORE_FIELD); PARSER.declareObjectArray( ConstructingObjectParser.optionalConstructorArg(), diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java index e9d32594fa833..5a6217eea8f7b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java @@ -14,7 +14,9 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -27,11 +29,13 @@ * supports the following configurations only explicitly enabling or disabling the failure store */ public record DataStreamFailureStore(Boolean enabled) implements SimpleDiffable, ToXContentObject { + public static final String FAILURE_STORE = "failure_store"; + public static final String ENABLED = "enabled"; - public static final ParseField ENABLED_FIELD = new ParseField("enabled"); + public static final ParseField ENABLED_FIELD = new ParseField(ENABLED); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "failure_store", + FAILURE_STORE, false, (args, unused) -> new DataStreamFailureStore((Boolean) args[0]) ); @@ -59,13 +63,6 @@ public static Diff readDiffFrom(StreamInput in) throws I return SimpleDiffable.readDiffFrom(DataStreamFailureStore::new, in); } - /** - * @return iff the user has explicitly enabled the failure store - */ - public boolean isExplicitlyEnabled() { - return enabled != null && enabled; - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalBoolean(enabled); @@ -89,4 +86,80 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static DataStreamFailureStore fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } + + /** + * This class is only used in template configuration. It wraps the fields of {@link DataStreamFailureStore} with {@link ResettableValue} + * to allow a user to signal when they want to reset any previously encountered values during template composition. Furthermore, it + * provides the method {@link #merge(Template, Template)} that dictates how two templates can be composed. + */ + public record Template(ResettableValue enabled) implements Writeable, ToXContentObject { + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "failure_store_template", + false, + (args, unused) -> new Template(args[0] == null ? ResettableValue.undefined() : (ResettableValue) args[0]) + ); + + static { + PARSER.declareField( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> p.currentToken() == XContentParser.Token.VALUE_NULL + ? ResettableValue.reset() + : ResettableValue.create(p.booleanValue()), + ENABLED_FIELD, + ObjectParser.ValueType.BOOLEAN_OR_NULL + ); + } + + public Template { + if (enabled.get() == null) { + throw new IllegalArgumentException("Failure store configuration should have at least one non-null configuration value."); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + ResettableValue.write(out, enabled, StreamOutput::writeBoolean); + } + + public static Template read(StreamInput in) throws IOException { + ResettableValue enabled = ResettableValue.read(in, StreamInput::readBoolean); + return new Template(enabled); + } + + /** + * Converts the template to XContent, depending on the XContent.Params set by {@link ResettableValue#hideResetValues(Params)} + * it may or may not display any explicit nulls when the value is to be reset. + */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + enabled.toXContent(builder, params, ENABLED_FIELD.getPreferredName()); + builder.endObject(); + return builder; + } + + public static Template fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + /** + * Returns a template which has the value of the initial template updated with the values of the update. + * Note: for now it's a trivial composition because we have only one non-null field. + * @return the composed template + */ + public static Template merge(Template ignored, Template update) { + return update; + } + + public DataStreamFailureStore toFailureStore() { + return new DataStreamFailureStore(enabled.get()); + } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java index 9cd4e2625e2ba..51e13c05e6892 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java @@ -14,9 +14,9 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.ConstructingObjectParser; -import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -24,6 +24,8 @@ import java.io.IOException; +import static org.elasticsearch.cluster.metadata.DataStreamFailureStore.FAILURE_STORE; + /** * Holds data stream dedicated configuration options such as failure store, (in the future lifecycle). Currently, it * supports the following configurations: @@ -34,10 +36,10 @@ public record DataStreamOptions(@Nullable DataStreamFailureStore failureStore) SimpleDiffable, ToXContentObject { - public static final ParseField FAILURE_STORE_FIELD = new ParseField("failure_store"); + public static final ParseField FAILURE_STORE_FIELD = new ParseField(FAILURE_STORE); public static final DataStreamOptions FAILURE_STORE_ENABLED = new DataStreamOptions(new DataStreamFailureStore(true)); public static final DataStreamOptions FAILURE_STORE_DISABLED = new DataStreamOptions(new DataStreamFailureStore(false)); - public static final DataStreamOptions EMPTY = new DataStreamOptions(); + public static final DataStreamOptions EMPTY = new DataStreamOptions(null); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "options", @@ -46,18 +48,13 @@ public record DataStreamOptions(@Nullable DataStreamFailureStore failureStore) ); static { - PARSER.declareField( + PARSER.declareObject( ConstructingObjectParser.optionalConstructorArg(), (p, c) -> DataStreamFailureStore.fromXContent(p), - FAILURE_STORE_FIELD, - ObjectParser.ValueType.OBJECT_OR_NULL + FAILURE_STORE_FIELD ); } - public DataStreamOptions() { - this(null); - } - public static DataStreamOptions read(StreamInput in) throws IOException { return new DataStreamOptions(in.readOptionalWriteable(DataStreamFailureStore::new)); } @@ -66,8 +63,21 @@ public static Diff readDiffFrom(StreamInput in) throws IOExce return SimpleDiffable.readDiffFrom(DataStreamOptions::read, in); } + /** + * @return true if none of the options are defined + */ public boolean isEmpty() { - return this.equals(EMPTY); + return failureStore == null; + } + + /** + * Determines if this data stream has its failure store enabled or not. Currently, the failure store + * is enabled only when a user has explicitly requested it. + * + * @return true, if the user has explicitly enabled the failure store. + */ + public boolean isFailureStoreEnabled() { + return failureStore != null && Boolean.TRUE.equals(failureStore.enabled()); } @Override @@ -93,4 +103,100 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static DataStreamOptions fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } + + /** + * This class is only used in template configuration. It wraps the fields of {@link DataStreamOptions} with {@link ResettableValue} + * to allow a user to signal when they want to reset any previously encountered values during template composition. Furthermore, it + * provides the {@link Template.Builder} that dictates how two templates can be composed. + */ + public record Template(ResettableValue failureStore) implements Writeable, ToXContentObject { + public static final Template EMPTY = new Template(ResettableValue.undefined()); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "data_stream_options_template", + false, + (args, unused) -> new Template( + args[0] == null ? ResettableValue.undefined() : (ResettableValue) args[0] + ) + ); + + static { + PARSER.declareObjectOrNull( + ConstructingObjectParser.optionalConstructorArg(), + (p, s) -> ResettableValue.create(DataStreamFailureStore.Template.fromXContent(p)), + ResettableValue.reset(), + FAILURE_STORE_FIELD + ); + } + + public Template { + assert failureStore != null : "Template does not accept null values, please use Resettable.undefined()"; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + ResettableValue.write(out, failureStore, (o, v) -> v.writeTo(o)); + } + + public static Template read(StreamInput in) throws IOException { + ResettableValue failureStore = ResettableValue.read(in, DataStreamFailureStore.Template::read); + return new Template(failureStore); + } + + public static Template fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + /** + * Converts the template to XContent, depending on the {@param params} set by {@link ResettableValue#hideResetValues(Params)} + * it may or may not display any explicit nulls when the value is to be reset. + */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + failureStore.toXContent(builder, params, FAILURE_STORE_FIELD.getPreferredName()); + builder.endObject(); + return builder; + } + + public DataStreamOptions toDataStreamOptions() { + return new DataStreamOptions(failureStore.mapAndGet(DataStreamFailureStore.Template::toFailureStore)); + } + + public static Builder builder(Template template) { + return new Builder(template); + } + + /** + * Builds and composes a data stream template. + */ + public static class Builder { + private ResettableValue failureStore = ResettableValue.undefined(); + + public Builder(Template template) { + if (template != null) { + failureStore = template.failureStore(); + } + } + + /** + * Updates the current failure store configuration with the provided value. This is not a replacement necessarily, if both + * instance contain data the configurations are merged. + */ + public Builder updateFailureStore(ResettableValue newFailureStore) { + failureStore = ResettableValue.merge(failureStore, newFailureStore, DataStreamFailureStore.Template::merge); + return this; + } + + public Template build() { + return new Template(failureStore); + } + } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 5dbf4da6f376f..0de87c7226380 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -261,11 +261,16 @@ static ClusterState createDataStream( // This is not a problem as both have different prefixes (`.ds-` vs `.fs-`) and both will be using the same `generation` field // when rolling over in the future. final long initialGeneration = 1; + ResettableValue dataStreamOptionsTemplate = isSystem + ? MetadataIndexTemplateService.resolveDataStreamOptions(template, systemDataStreamDescriptor.getComponentTemplates()) + : MetadataIndexTemplateService.resolveDataStreamOptions(template, metadata.componentTemplates()); + final DataStreamOptions dataStreamOptions = dataStreamOptionsTemplate.mapAndGet(DataStreamOptions.Template::toDataStreamOptions); + var isFailureStoreEnabled = dataStreamOptions != null && dataStreamOptions.isFailureStoreEnabled(); // If we need to create a failure store, do so first. Do not reroute during the creation since we will do // that as part of creating the backing index if required. IndexMetadata failureStoreIndex = null; - if (template.getDataStreamTemplate().hasFailureStore() && initializeFailureStore) { + if (isFailureStoreEnabled && initializeFailureStore) { if (isSystem) { throw new IllegalArgumentException("Failure stores are not supported on system data streams"); } @@ -303,7 +308,7 @@ static ClusterState createDataStream( } assert writeIndex != null; assert writeIndex.mapping() != null : "no mapping found for backing index [" + writeIndex.getIndex().getName() + "]"; - assert template.getDataStreamTemplate().hasFailureStore() == false || initializeFailureStore == false || failureStoreIndex != null + assert isFailureStoreEnabled == false || initializeFailureStore == false || failureStoreIndex != null : "failure store should have an initial index"; assert failureStoreIndex == null || failureStoreIndex.mapping() != null : "no mapping found for failure store [" + failureStoreIndex.getIndex().getName() + "]"; @@ -329,7 +334,7 @@ static ClusterState createDataStream( template.getDataStreamTemplate().isAllowCustomRouting(), indexMode, lifecycle == null && isDslOnlyMode ? DataStreamLifecycle.DEFAULT : lifecycle, - template.getDataStreamTemplate().hasFailureStore() ? DataStreamOptions.FAILURE_STORE_ENABLED : DataStreamOptions.EMPTY, + dataStreamOptions, new DataStream.DataStreamIndices(DataStream.BACKING_INDEX_PREFIX, dsBackingIndices, false, null), // If the failure store shouldn't be initialized on data stream creation, we're marking it for "lazy rollover", which will // initialize the failure store on first write. diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 3878a3329b634..7f8b87d2d3f48 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -312,12 +312,7 @@ public ClusterState addComponentTemplate( } } - final Template finalTemplate = new Template( - finalSettings, - wrappedMappings, - template.template().aliases(), - template.template().lifecycle() - ); + final Template finalTemplate = Template.builder(template.template()).settings(finalSettings).mappings(wrappedMappings).build(); final ComponentTemplate finalComponentTemplate = new ComponentTemplate( finalTemplate, template.version(), @@ -348,6 +343,7 @@ public ClusterState addComponentTemplate( composableTemplate, globalRetentionSettings.get() ); + validateDataStreamOptions(tempStateWithComponentTemplateAdded.metadata(), composableTemplateName, composableTemplate); validateIndexTemplateV2(composableTemplateName, composableTemplate, tempStateWithComponentTemplateAdded); } catch (Exception e) { if (validationFailure == null) { @@ -629,7 +625,7 @@ public ClusterState addIndexTemplateV2( // adjusted (to add _doc) and it should be validated CompressedXContent mappings = innerTemplate.mappings(); CompressedXContent wrappedMappings = wrapMappingsIfNecessary(mappings, xContentRegistry); - final Template finalTemplate = new Template(finalSettings, wrappedMappings, innerTemplate.aliases(), innerTemplate.lifecycle()); + final Template finalTemplate = Template.builder(innerTemplate).settings(finalSettings).mappings(wrappedMappings).build(); finalIndexTemplate = template.toBuilder().template(finalTemplate).build(); } @@ -690,7 +686,8 @@ public static Map> v2TemplateOverlaps( return overlaps; } - private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexTemplate, ClusterState currentState) { + // Visibility for testing + void validateIndexTemplateV2(String name, ComposableIndexTemplate indexTemplate, ClusterState currentState) { // Workaround for the fact that start_time and end_time are injected by the MetadataCreateDataStreamService upon creation, // but when validating templates that create data streams the MetadataCreateDataStreamService isn't used. var finalTemplate = indexTemplate.template(); @@ -726,6 +723,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT validate(name, templateToValidate); validateDataStreamsStillReferenced(currentState, name, templateToValidate); validateLifecycle(currentState.metadata(), name, templateToValidate, globalRetentionSettings.get()); + validateDataStreamOptions(currentState.metadata(), name, templateToValidate); if (templateToValidate.isDeprecated() == false) { validateUseOfDeprecatedComponentTemplates(name, templateToValidate, currentState.metadata().componentTemplates()); @@ -819,6 +817,20 @@ static void validateLifecycle( } } + // Visible for testing + static void validateDataStreamOptions(Metadata metadata, String indexTemplateName, ComposableIndexTemplate template) { + ResettableValue dataStreamOptions = resolveDataStreamOptions(template, metadata.componentTemplates()); + if (dataStreamOptions.get() != null) { + if (template.getDataStreamTemplate() == null) { + throw new IllegalArgumentException( + "index template [" + + indexTemplateName + + "] specifies data stream options that can only be used in combination with a data stream" + ); + } + } + } + /** * Validate that by changing or adding {@code newTemplate}, there are * no unreferenced data streams. Note that this scenario is still possible @@ -1561,7 +1573,7 @@ static List> resolveAliases( public static DataStreamLifecycle resolveLifecycle(final Metadata metadata, final String templateName) { final ComposableIndexTemplate template = metadata.templatesV2().get(templateName); assert template != null - : "attempted to resolve settings for a template [" + templateName + "] that did not exist in the cluster state"; + : "attempted to resolve lifecycle for a template [" + templateName + "] that did not exist in the cluster state"; if (template == null) { return null; } @@ -1653,6 +1665,81 @@ public static DataStreamLifecycle composeDataLifecycles(List} object + */ + public static ResettableValue resolveDataStreamOptions(final Metadata metadata, final String templateName) { + final ComposableIndexTemplate template = metadata.templatesV2().get(templateName); + assert template != null + : "attempted to resolve data stream options for a template [" + templateName + "] that did not exist in the cluster state"; + if (template == null) { + return ResettableValue.undefined(); + } + return resolveDataStreamOptions(template, metadata.componentTemplates()); + } + + /** + * Resolve the provided v2 template and component templates into a {@link ResettableValue} object + */ + public static ResettableValue resolveDataStreamOptions( + ComposableIndexTemplate template, + Map componentTemplates + ) { + Objects.requireNonNull(template, "attempted to resolve data stream for a null template"); + Objects.requireNonNull(componentTemplates, "attempted to resolve data stream options with null component templates"); + + List> dataStreamOptionsList = new ArrayList<>(); + for (String componentTemplateName : template.composedOf()) { + if (componentTemplates.containsKey(componentTemplateName) == false) { + continue; + } + ResettableValue dataStreamOptions = componentTemplates.get(componentTemplateName) + .template() + .resettableDataStreamOptions(); + if (dataStreamOptions.isDefined()) { + dataStreamOptionsList.add(dataStreamOptions); + } + } + // The actual index template's data stream options have the highest precedence. + if (template.template() != null && template.template().resettableDataStreamOptions().isDefined()) { + dataStreamOptionsList.add(template.template().resettableDataStreamOptions()); + } + return composeDataStreamOptions(dataStreamOptionsList); + } + + /** + * This method composes a series of data streams options to a final one. Since currently the data stream options + * contains only the failure store configuration which also contains only one field, the composition is a bit trivial. + * But we introduce the mechanics that will help extend it really easily. + * @param dataStreamOptionsList a sorted list of data stream options in the order that they will be composed + * @return the final data stream option configuration + */ + public static ResettableValue composeDataStreamOptions( + List> dataStreamOptionsList + ) { + if (dataStreamOptionsList.isEmpty()) { + return ResettableValue.undefined(); + } + DataStreamOptions.Template.Builder builder = null; + for (ResettableValue current : dataStreamOptionsList) { + if (current.isDefined() == false) { + continue; + } + if (current.shouldReset()) { + builder = null; + } else { + DataStreamOptions.Template currentTemplate = current.get(); + if (builder == null) { + builder = DataStreamOptions.Template.builder(currentTemplate); + } else { + // Currently failure store has only one field that needs to be defined so the composing of the failure store is trivial + builder.updateFailureStore(currentTemplate.failureStore()); + } + } + } + return builder == null ? ResettableValue.undefined() : ResettableValue.create(builder.build()); + } + /** * Given a state and a composable template, validate that the final composite template * generated by the composable template and all of its component templates contains valid diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ResettableValue.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ResettableValue.java new file mode 100644 index 0000000000000..4f38d2b8386a6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ResettableValue.java @@ -0,0 +1,216 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; +import java.util.function.BiFunction; +import java.util.function.Function; + +/** + * This class holds a value of type @{param T} that can be in one of 3 states: + * - It has a concrete value, or + * - It is missing, or + * - It is meant to reset any other when it is composed with it. + * It is mainly used in template composition to capture the case when the user wished to reset any previous values. + * @param + */ +public class ResettableValue { + private static final ResettableValue RESET = new ResettableValue<>(true, null); + private static final ResettableValue UNDEFINED = new ResettableValue<>(false, null); + private static final String DISPLAY_RESET_VALUES = "display_reset"; + private static final Map HIDE_RESET_VALUES_PARAMS = Map.of(DISPLAY_RESET_VALUES, "false"); + + private final T value; + private final boolean isDefined; + + /** + * @return the reset state, meaning that this value is explicitly requested to be reset + */ + public static ResettableValue reset() { + @SuppressWarnings("unchecked") + ResettableValue t = (ResettableValue) RESET; + return t; + } + + /** + * @return the undefined state, meaning that this value has not been specified + */ + public static ResettableValue undefined() { + @SuppressWarnings("unchecked") + ResettableValue t = (ResettableValue) UNDEFINED; + return t; + } + + /** + * Wraps a value, if the value is null, it returns {@link #undefined()} + */ + public static ResettableValue create(T value) { + if (value == null) { + return undefined(); + } + return new ResettableValue<>(true, value); + } + + private ResettableValue(boolean isDefined, T value) { + this.isDefined = isDefined; + this.value = value; + } + + /** + * @return true if the state of this is reset + */ + public boolean shouldReset() { + return isDefined && value == null; + } + + /** + * @return true when the value is defined, either with a concrete value or reset. + */ + public boolean isDefined() { + return isDefined; + } + + /** + * @return the concrete value or null if it is in undefined or reset states. + */ + @Nullable + public T get() { + return value; + } + + /** + * Writes a single optional explicitly nullable value. This method is in direct relation with the + * {@link #read(StreamInput, Writeable.Reader)} which reads the respective value. It's the + * responsibility of the caller to preserve order of the fields and their backwards compatibility. + * + * @throws IOException + */ + static void write(StreamOutput out, ResettableValue value, Writeable.Writer writer) throws IOException { + out.writeBoolean(value.isDefined); + if (value.isDefined) { + out.writeBoolean(value.shouldReset()); + if (value.shouldReset() == false) { + writer.write(out, value.get()); + } + } + } + + /** + * Reads a single optional and explicitly nullable value. This method is in direct relation with the + * {@link #write(StreamOutput, ResettableValue, Writeable.Writer)} which writes the respective value. It's the + * responsibility of the caller to preserve order of the fields and their backwards compatibility. + * + * @throws IOException + */ + @Nullable + static ResettableValue read(StreamInput in, Writeable.Reader reader) throws IOException { + boolean isDefined = in.readBoolean(); + if (isDefined == false) { + return ResettableValue.undefined(); + } + boolean shouldReset = in.readBoolean(); + if (shouldReset) { + return ResettableValue.reset(); + } + T value = reader.read(in); + return ResettableValue.create(value); + } + + /** + * Gets the value and applies the function {@param f} when the value is not null. Slightly more efficient than + * this.map(f).get(). + */ + public U mapAndGet(Function f) { + if (isDefined() == false || shouldReset()) { + return null; + } else { + return f.apply(value); + } + } + + public ResettableValue map(Function mapper) { + Objects.requireNonNull(mapper); + if (isDefined == false) { + return ResettableValue.undefined(); + } + if (shouldReset()) { + return reset(); + } + return ResettableValue.create(mapper.apply(value)); + } + + /** + * Ιt merges the values of the ResettableValue's when they are defined using the provided mergeFunction. + */ + public static ResettableValue merge(ResettableValue initial, ResettableValue update, BiFunction mergeFunction) { + if (update.shouldReset()) { + return undefined(); + } + if (update.isDefined() == false) { + return initial; + } + if (initial.isDefined() == false || initial.shouldReset()) { + return update; + } + // Because we checked that's defined and not in reset state, we can directly apply the merge function. + return ResettableValue.create(mergeFunction.apply(initial.value, update.value)); + } + + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params, String field) throws IOException { + return toXContent(builder, params, field, Function.identity()); + } + + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params, String field, Function transformValue) + throws IOException { + if (isDefined) { + if (value != null) { + builder.field(field, transformValue.apply(value)); + } else if (ResettableValue.shouldDisplayResetValue(params)) { + builder.nullField(field); + } + } + return builder; + } + + public static boolean shouldDisplayResetValue(ToXContent.Params params) { + return params.paramAsBoolean(DISPLAY_RESET_VALUES, true); + } + + public static ToXContent.Params hideResetValues(ToXContent.Params params) { + return new ToXContent.DelegatingMapParams(HIDE_RESET_VALUES_PARAMS, params); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ResettableValue that = (ResettableValue) o; + return isDefined == that.isDefined && Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(value, isDefined); + } + + @Override + public String toString() { + return "ResettableValue{" + "value=" + value + ", isDefined=" + isDefined + '}'; + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java index 0a9e79284ced6..7d354768ca987 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java @@ -47,12 +47,19 @@ public class Template implements SimpleDiffable