From 5cfa702ce6579a75b66c42181ae66f479a8efecb Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 21 Mar 2023 11:34:57 -0700 Subject: [PATCH] Rework on `OpenSearchDataType`: parse, store and use mapping information (#1314) (#1455) * Rework on `OpenSearchDataType`: parse, store and use mapping information (#180) Rework on `OpenSearchDataType`: * Add data types for those classes are not defined in `ExprCoreType`. * Address https://github.com/Bit-Quill/opensearch-project-sql/pull/180#discussion_r1044941128 * Remove `TextKeywordValue`. * Add changes according to the PR review. https://github.com/Bit-Quill/opensearch-project-sql/pull/180 * Update `IndexMapping::parseMapping` function. * Add `OpenSearchDataType::resolve` function. * Add new constructor for `OpenSearchTextType`. * Make `fields` and `properties` in `OpenSearchDataType` readonly. Update tests and mapping parser. * Move `getFields` from `OpenSearchDataType` to `OpenSearchTextType`. Update tests. * Rewrite `traverseAndFlatten` according to https://github.com/Bit-Quill/opensearch-project-sql/pull/180#discussion_r1083110098 * Minor comment fix. * A fix to avoid breaking changes. * `typeName` and `legacyTypeName` to return different type names. * Change `typeof` function and corresponding tests. * Move `convertTextToKeyword` from `ScriptUtils` to `OpenSearchTextType`. Update tests. * Update UT for `typeof` function. * Make all instances of `OpenSearchDataType` and of derived types singletones as much as possible. * Make string representations of all `ExprType`s uppercase. * Remove functions from `IndexMapping` used in tests only. Signed-off-by: Yury-Fridlyand Signed-off-by: Yury-Fridlyand (cherry picked from commit d44cd39c34367d96715731adede370794ac0d95a) Co-authored-by: Yury-Fridlyand --- .../sql/data/model/AbstractExprValue.java | 2 +- .../sql/data/type/ExprCoreType.java | 6 +- .../expression/system/SystemFunctions.java | 2 +- .../sql/data/type/ExprTypeTest.java | 6 +- .../system/SystemFunctionsTest.java | 6 +- docs/user/dql/functions.rst | 2 +- docs/user/general/datatypes.rst | 4 +- docs/user/ppl/functions/system.rst | 2 +- docs/user/ppl/general/datatypes.rst | 4 +- .../opensearch/sql/ppl/SystemFunctionIT.java | 6 +- .../opensearch/sql/sql/SystemFunctionIT.java | 6 +- .../data/type/OpenSearchBinaryType.java | 35 ++ .../data/type/OpenSearchDataType.java | 305 +++++++++----- .../data/type/OpenSearchGeoPointType.java | 35 ++ .../data/type/OpenSearchIpType.java | 35 ++ .../data/type/OpenSearchTextType.java | 76 ++++ .../data/value/OpenSearchExprBinaryValue.java | 5 +- .../value/OpenSearchExprGeoPointValue.java | 5 +- .../data/value/OpenSearchExprIpValue.java | 5 +- .../value/OpenSearchExprTextKeywordValue.java | 30 -- .../data/value/OpenSearchExprTextValue.java | 7 +- .../value/OpenSearchExprValueFactory.java | 116 +++--- .../sql/opensearch/mapping/IndexMapping.java | 98 ++--- .../request/OpenSearchRequestBuilder.java | 6 +- .../OpenSearchDescribeIndexRequest.java | 20 +- .../opensearch/storage/OpenSearchIndex.java | 37 +- .../storage/script/ScriptUtils.java | 30 -- .../aggregation/AggregationQueryBuilder.java | 14 +- .../dsl/AggregationBuilderHelper.java | 5 +- .../storage/script/core/ExpressionScript.java | 20 +- .../script/filter/lucene/LikeQuery.java | 3 +- .../script/filter/lucene/LuceneQuery.java | 19 +- .../script/filter/lucene/TermQuery.java | 4 +- .../storage/script/sort/SortQueryBuilder.java | 5 +- .../client/OpenSearchNodeClientTest.java | 86 +++- .../client/OpenSearchRestClientTest.java | 88 +++- .../OpenSearchDataTypeRecognitionTest.java | 31 +- .../data/type/OpenSearchDataTypeTest.java | 384 ++++++++++++++++-- .../value/OpenSearchExprBinaryValueTest.java | 5 +- .../OpenSearchExprGeoPointValueTest.java | 11 +- .../data/value/OpenSearchExprIpValueTest.java | 5 +- .../OpenSearchExprTextKeywordValueTest.java | 21 - .../value/OpenSearchExprTextValueTest.java | 67 ++- .../value/OpenSearchExprValueFactoryTest.java | 103 +++-- .../opensearch/mapping/IndexMappingTest.java | 37 -- .../request/OpenSearchRequestBuilderTest.java | 5 +- .../OpenSearchDescribeIndexRequestTest.java | 35 +- .../storage/OpenSearchIndexScanTest.java | 5 +- .../storage/OpenSearchIndexTest.java | 85 ++-- .../OpenSearchIndexScanOptimizationTest.java | 6 +- .../AggregationQueryBuilderTest.java | 26 +- .../ExpressionAggregationScriptTest.java | 11 +- .../dsl/BucketAggregationBuilderTest.java | 7 +- .../filter/ExpressionFilterScriptTest.java | 7 +- .../script/filter/FilterQueryBuilderTest.java | 49 ++- .../lucene/MatchBoolPrefixQueryTest.java | 8 +- .../lucene/MatchPhrasePrefixQueryTest.java | 18 +- .../filter/lucene/MatchPhraseQueryTest.java | 44 +- .../script/filter/lucene/MatchQueryTest.java | 36 +- .../filter/lucene/WildcardQueryTest.java | 8 +- .../relevance/SingleFieldQueryTest.java | 7 +- .../format/JdbcResponseFormatterTest.java | 10 +- 62 files changed, 1444 insertions(+), 722 deletions(-) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchBinaryType.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchGeoPointType.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchIpType.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextKeywordValue.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/ScriptUtils.java delete mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextKeywordValueTest.java delete mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/mapping/IndexMappingTest.java diff --git a/core/src/main/java/org/opensearch/sql/data/model/AbstractExprValue.java b/core/src/main/java/org/opensearch/sql/data/model/AbstractExprValue.java index 6a823dff68..ad2c2ddb49 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/AbstractExprValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/AbstractExprValue.java @@ -23,7 +23,7 @@ public int compareTo(ExprValue other) { } if ((this.isNumber() && other.isNumber()) || (this.isDateTime() && other.isDateTime()) - || this.type() == other.type()) { + || this.type().equals(other.type())) { return compare(other); } else { throw new ExpressionEvaluationException( diff --git a/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java b/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java index a9ceeac4dc..815f94a9df 100644 --- a/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java +++ b/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java @@ -80,9 +80,9 @@ public enum ExprCoreType implements ExprType { */ private static final Map LEGACY_TYPE_NAME_MAPPING = new ImmutableMap.Builder() - .put(STRUCT, "object") - .put(ARRAY, "nested") - .put(STRING, "keyword") + .put(STRUCT, "OBJECT") + .put(ARRAY, "NESTED") + .put(STRING, "KEYWORD") .build(); private static final Set NUMBER_TYPES = diff --git a/core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java b/core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java index 6d8dd5093e..e12bcd0a58 100644 --- a/core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java @@ -42,7 +42,7 @@ public Pair resolve( new FunctionExpression(BuiltinFunctionName.TYPEOF.getName(), arguments) { @Override public ExprValue valueOf(Environment valueEnv) { - return new ExprStringValue(getArguments().get(0).type().toString()); + return new ExprStringValue(getArguments().get(0).type().legacyTypeName()); } @Override diff --git a/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java b/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java index 204ca197e1..7db856d092 100644 --- a/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java +++ b/core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java @@ -76,9 +76,9 @@ public void getParent() { @Test void legacyName() { - assertEquals("keyword", STRING.legacyTypeName()); - assertEquals("nested", ARRAY.legacyTypeName()); - assertEquals("object", STRUCT.legacyTypeName()); + assertEquals("KEYWORD", STRING.legacyTypeName()); + assertEquals("NESTED", ARRAY.legacyTypeName()); + assertEquals("OBJECT", STRUCT.legacyTypeName()); assertEquals("integer", INTEGER.legacyTypeName().toLowerCase()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java index cdf8e6ec31..62d219e576 100644 --- a/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java @@ -45,7 +45,7 @@ public class SystemFunctionsTest { void typeof() { assertEquals(STRING, DSL.typeof(DSL.literal(1)).type()); - assertEquals("ARRAY", typeofGetValue(new ExprCollectionValue(List.of()))); + assertEquals("NESTED", typeofGetValue(new ExprCollectionValue(List.of()))); assertEquals("BOOLEAN", typeofGetValue(ExprBooleanValue.of(false))); assertEquals("BYTE", typeofGetValue(new ExprByteValue(0))); assertEquals("DATE", typeofGetValue(new ExprDateValue(LocalDate.now()))); @@ -56,8 +56,8 @@ void typeof() { assertEquals("INTERVAL", typeofGetValue(new ExprIntervalValue(Duration.ofDays(0)))); assertEquals("LONG", typeofGetValue(new ExprLongValue(0))); assertEquals("SHORT", typeofGetValue(new ExprShortValue(0))); - assertEquals("STRING", typeofGetValue(new ExprStringValue(""))); - assertEquals("STRUCT", typeofGetValue(new ExprTupleValue(new LinkedHashMap<>()))); + assertEquals("KEYWORD", typeofGetValue(new ExprStringValue(""))); + assertEquals("OBJECT", typeofGetValue(new ExprTupleValue(new LinkedHashMap<>()))); assertEquals("TIME", typeofGetValue(new ExprTimeValue(LocalTime.now()))); assertEquals("TIMESTAMP", typeofGetValue(new ExprTimestampValue(Instant.now()))); assertEquals("UNDEFINED", typeofGetValue(ExprNullValue.of())); diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 650c322c3b..eb10240ab9 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -4286,7 +4286,7 @@ Example:: +----------------+---------------+-----------------+------------------+ | typeof(date) | typeof(int) | typeof(now()) | typeof(column) | |----------------+---------------+-----------------+------------------| - | DATE | INTEGER | DATETIME | STRUCT | + | DATE | INTEGER | DATETIME | OBJECT | +----------------+---------------+-----------------+------------------+ diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index 63b57f87e8..5899488ab0 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -34,7 +34,7 @@ The OpenSearch SQL Engine support the following data types. +---------------------+ | double | +---------------------+ -| string | +| keyword | +---------------------+ | text | +---------------------+ @@ -87,7 +87,7 @@ The table below list the mapping between OpenSearch Data Type, OpenSearch SQL Da +-----------------+---------------------+-----------+ | double | double | DOUBLE | +-----------------+---------------------+-----------+ -| keyword | string | VARCHAR | +| keyword | keyword | VARCHAR | +-----------------+---------------------+-----------+ | text | text | VARCHAR | +-----------------+---------------------+-----------+ diff --git a/docs/user/ppl/functions/system.rst b/docs/user/ppl/functions/system.rst index 65585c740a..fbe9860dce 100644 --- a/docs/user/ppl/functions/system.rst +++ b/docs/user/ppl/functions/system.rst @@ -27,5 +27,5 @@ Example:: +----------------+---------------+-----------------+------------------+ | typeof(date) | typeof(int) | typeof(now()) | typeof(column) | |----------------+---------------+-----------------+------------------| - | DATE | INTEGER | DATETIME | STRUCT | + | DATE | INTEGER | DATETIME | OBJECT | +----------------+---------------+-----------------+------------------+ diff --git a/docs/user/ppl/general/datatypes.rst b/docs/user/ppl/general/datatypes.rst index c7478ae62d..cabc689526 100644 --- a/docs/user/ppl/general/datatypes.rst +++ b/docs/user/ppl/general/datatypes.rst @@ -37,8 +37,6 @@ The PPL support the following data types. +---------------+ | string | +---------------+ -| text | -+---------------+ | timestamp | +---------------+ | datetime | @@ -88,7 +86,7 @@ The table below list the mapping between OpenSearch Data Type, PPL Data Type and +-----------------+---------------+-----------+ | keyword | string | VARCHAR | +-----------------+---------------+-----------+ -| text | text | VARCHAR | +| text | string | VARCHAR | +-----------------+---------------+-----------+ | date | timestamp | TIMESTAMP | +-----------------+---------------+-----------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java index 7e8383baa4..de13aa5488 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -33,7 +33,7 @@ public void typeof_sql_types() throws IOException { TEST_INDEX_DATATYPE_NUMERIC)); // TODO: test null in PPL verifyDataRows(response, - rows("STRING", "DOUBLE", "INTEGER", "LONG", "INTERVAL")); + rows("KEYWORD", "DOUBLE", "INTEGER", "LONG", "INTERVAL")); response = executeQuery(String.format("source=%s | eval " + "`timestamp` = typeof(CAST('1961-04-12 09:07:00' AS TIMESTAMP))," @@ -68,7 +68,7 @@ public void typeof_opensearch_types() throws IOException { + " | fields `text`, `date`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`", TEST_INDEX_DATATYPE_NONNUMERIC)); verifyDataRows(response, - rows("OPENSEARCH_TEXT", "TIMESTAMP", "BOOLEAN", "STRUCT", "STRING", - "OPENSEARCH_IP", "OPENSEARCH_BINARY", "OPENSEARCH_GEO_POINT")); + rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD", + "IP", "BINARY", "GEO_POINT")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java index 0b43ec0479..584cdd05dd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java @@ -27,7 +27,7 @@ public void typeof_sql_types() { JSONObject response = executeJdbcRequest("SELECT typeof('pewpew'), typeof(NULL), typeof(1.0)," + "typeof(12345), typeof(1234567891011), typeof(INTERVAL 2 DAY);"); verifyDataRows(response, - rows("STRING", "UNDEFINED", "DOUBLE", "INTEGER", "LONG", "INTERVAL")); + rows("KEYWORD", "UNDEFINED", "DOUBLE", "INTEGER", "LONG", "INTERVAL")); response = executeJdbcRequest("SELECT" + " typeof(CAST('1961-04-12 09:07:00' AS TIMESTAMP))," @@ -54,7 +54,7 @@ public void typeof_opensearch_types() { //+ ", typeof(nested_value)" + " from %s;", TEST_INDEX_DATATYPE_NONNUMERIC)); verifyDataRows(response, - rows("OPENSEARCH_TEXT", "TIMESTAMP", "BOOLEAN", "STRUCT", "STRING", - "OPENSEARCH_IP", "OPENSEARCH_BINARY", "OPENSEARCH_GEO_POINT")); + rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD", + "IP", "BINARY", "GEO_POINT")); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchBinaryType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchBinaryType.java new file mode 100644 index 0000000000..cd58d4bc9f --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchBinaryType.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.opensearch.data.type; + +import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; + +import lombok.EqualsAndHashCode; + +/** + * The type of a binary value. See + * doc + */ +@EqualsAndHashCode(callSuper = false) +public class OpenSearchBinaryType extends OpenSearchDataType { + + private static final OpenSearchBinaryType instance = new OpenSearchBinaryType(); + + private OpenSearchBinaryType() { + super(MappingType.Binary); + exprCoreType = UNKNOWN; + } + + public static OpenSearchBinaryType of() { + return OpenSearchBinaryType.instance; + } + + @Override + protected OpenSearchDataType cloneEmpty() { + return instance; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 05b80bfa23..2fda12a567 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -6,139 +6,260 @@ package org.opensearch.sql.opensearch.data.type; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; - -import com.google.common.collect.BiMap; -import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableMap; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.io.Serializable; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; -import lombok.RequiredArgsConstructor; +import java.util.function.BiConsumer; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.commons.lang3.EnumUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; /** - * The extension of ExprType in Elasticsearch. + * The extension of ExprType in OpenSearch. */ -@RequiredArgsConstructor -public enum OpenSearchDataType implements ExprType { - /** - * OpenSearch Text. Rather than cast text to other types (STRING), leave it alone to prevent - * cast_to_string(OPENSEARCH_TEXT). - * Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/text.html - */ - OPENSEARCH_TEXT(Collections.singletonList(STRING), "string") { - @Override - public boolean shouldCast(ExprType other) { - return false; - } - }, +@EqualsAndHashCode +public class OpenSearchDataType implements ExprType, Serializable { /** - * OpenSearch multi-fields which has text and keyword. - * Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-fields.html + * The mapping (OpenSearch engine) type. */ - OPENSEARCH_TEXT_KEYWORD(Arrays.asList(STRING, OPENSEARCH_TEXT), "string") { - @Override - public boolean shouldCast(ExprType other) { - return false; - } - }, + public enum MappingType { + Invalid(null, ExprCoreType.UNKNOWN), + Text("text", ExprCoreType.UNKNOWN), + Keyword("keyword", ExprCoreType.STRING), + Ip("ip", ExprCoreType.UNKNOWN), + GeoPoint("geo_point", ExprCoreType.UNKNOWN), + Binary("binary", ExprCoreType.UNKNOWN), + Date("date", ExprCoreType.TIMESTAMP), + Object("object", ExprCoreType.STRUCT), + Nested("nested", ExprCoreType.ARRAY), + Byte("byte", ExprCoreType.BYTE), + Short("short", ExprCoreType.SHORT), + Integer("integer", ExprCoreType.INTEGER), + Long("long", ExprCoreType.LONG), + Float("float", ExprCoreType.FLOAT), + HalfFloat("half_float", ExprCoreType.FLOAT), + ScaledFloat("scaled_float", ExprCoreType.DOUBLE), + Double("double", ExprCoreType.DOUBLE), + Boolean("boolean", ExprCoreType.BOOLEAN); + // TODO: ranges, geo shape, point, shape + private final String name; - OPENSEARCH_IP(Arrays.asList(UNKNOWN), "ip"), + // Associated `ExprCoreType` + @Getter + private final ExprCoreType exprCoreType; - OPENSEARCH_GEO_POINT(Arrays.asList(UNKNOWN), "geo_point"), + MappingType(String name, ExprCoreType exprCoreType) { + this.name = name; + this.exprCoreType = exprCoreType; + } - OPENSEARCH_BINARY(Arrays.asList(UNKNOWN), "binary"); + public String toString() { + return name; + } + } - /** - * Bidirectional mapping between OpenSearch type name and ExprType. - */ - private static final BiMap OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING = - ImmutableBiMap.builder() - .put("text", OPENSEARCH_TEXT) - .put("text_keyword", OPENSEARCH_TEXT_KEYWORD) - .put("keyword", ExprCoreType.STRING) - .put("byte", ExprCoreType.BYTE) - .put("short", ExprCoreType.SHORT) - .put("integer", ExprCoreType.INTEGER) - .put("long", ExprCoreType.LONG) - .put("float", ExprCoreType.FLOAT) - .put("double", ExprCoreType.DOUBLE) - .put("boolean", ExprCoreType.BOOLEAN) - .put("nested", ExprCoreType.ARRAY) - .put("object", ExprCoreType.STRUCT) - .put("date", ExprCoreType.TIMESTAMP) - .put("ip", OPENSEARCH_IP) - .put("geo_point", OPENSEARCH_GEO_POINT) - .put("binary", OPENSEARCH_BINARY) - .build(); + @EqualsAndHashCode.Exclude + protected MappingType mappingType; - /** - * Mapping from extra OpenSearch type name which may map to same ExprType as above. - */ - private static final Map EXTRA_OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING = - ImmutableMap.builder() - .put("half_float", ExprCoreType.FLOAT) - .put("scaled_float", ExprCoreType.DOUBLE) - .put("date_nanos", ExprCoreType.TIMESTAMP) - .build(); + // resolved ExprCoreType + protected ExprCoreType exprCoreType; /** - * The mapping between Type and legacy JDBC type name. + * Get a simplified type {@link ExprCoreType} if possible. + * To avoid returning `UNKNOWN` for `OpenSearch*Type`s, e.g. for IP, returns itself. + * @return An {@link ExprType}. */ - private static final Map LEGACY_TYPE_NAME_MAPPING = - new ImmutableMap.Builder() - .put(OPENSEARCH_TEXT, "text") - .put(OPENSEARCH_TEXT_KEYWORD, "text") - .build(); + public ExprType getExprType() { + if (exprCoreType != ExprCoreType.UNKNOWN) { + return exprCoreType; + } + return this; + } /** - * Parent of current type. + * Simple instances of OpenSearchDataType are created once during entire SQL engine lifetime + * and cached there. This reduces memory usage and increases type comparison. + * Note: Types with non-empty fields and properties are not cached. */ - private final List parents; + private static final Map instances = new HashMap<>(); + + static { + EnumUtils.getEnumList(MappingType.class).stream() + .filter(t -> t != MappingType.Invalid).forEach(t -> + instances.put(t.toString(), OpenSearchDataType.of(t))); + EnumUtils.getEnumList(ExprCoreType.class).forEach(t -> + instances.put(t.toString(), OpenSearchDataType.of(t))); + } + /** - * JDBC type name. + * A constructor function which builds proper `OpenSearchDataType` for given mapping `Type`. + * @param mappingType A mapping type. + * @return An instance or inheritor of `OpenSearchDataType`. */ - private final String jdbcType; + public static OpenSearchDataType of(MappingType mappingType) { + var res = instances.getOrDefault(mappingType.toString(), null); + if (res != null) { + return res; + } + ExprCoreType exprCoreType = mappingType.getExprCoreType(); + if (exprCoreType == ExprCoreType.UNKNOWN) { + switch (mappingType) { + // TODO update these 2 below #1038 https://github.com/opensearch-project/sql/issues/1038 + case Text: return OpenSearchTextType.of(); + case GeoPoint: return OpenSearchGeoPointType.of(); + case Binary: return OpenSearchBinaryType.of(); + case Ip: return OpenSearchIpType.of(); + default: + throw new IllegalArgumentException(mappingType.toString()); + } + } + res = new OpenSearchDataType(mappingType); + res.exprCoreType = exprCoreType; + return res; + } /** - * Convert OpenSearch type string to ExprType. - * @param openSearchType OpenSearch type string - * @return expr type + * A constructor function which builds proper `OpenSearchDataType` for given mapping `Type`. + * Designed to be called by the mapping parser only (and tests). + * @param mappingType A mapping type. + * @param properties Properties to set. + * @param fields Fields to set. + * @return An instance or inheritor of `OpenSearchDataType`. */ - public static ExprType getExprType(String openSearchType) { - if (OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.containsKey(openSearchType)) { - return OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.get(openSearchType); + public static OpenSearchDataType of(MappingType mappingType, + Map properties, + Map fields) { + var res = of(mappingType); + if (!properties.isEmpty() || !fields.isEmpty()) { + // Clone to avoid changing the singleton instance. + res = res.cloneEmpty(); + res.properties = ImmutableMap.copyOf(properties); + res.fields = ImmutableMap.copyOf(fields); } - return EXTRA_OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.getOrDefault(openSearchType, UNKNOWN); + return res; + } + + protected OpenSearchDataType(MappingType mappingType) { + this.mappingType = mappingType; } /** - * Convert ExprType to OpenSearch type string. - * @param type expr type - * @return OpenSearch type string + * A constructor function which builds proper `OpenSearchDataType` for given {@link ExprType}. + * @param type A type. + * @return An instance of `OpenSearchDataType`. */ - public static String getOpenSearchType(ExprType type) { - return OPENSEARCH_TYPE_TO_EXPR_TYPE_MAPPING.inverse().get(type); + public static OpenSearchDataType of(ExprType type) { + if (type instanceof OpenSearchDataType) { + return (OpenSearchDataType) type; + } + var res = instances.getOrDefault(type.toString(), null); + if (res != null) { + return res; + } + return new OpenSearchDataType((ExprCoreType) type); } - @Override - public List getParent() { - return parents; + protected OpenSearchDataType(ExprCoreType type) { + this.exprCoreType = type; } + protected OpenSearchDataType() { + } + + // For datatypes with properties (example: object and nested types) + // a read-only collection + @Getter + @EqualsAndHashCode.Exclude + Map properties = ImmutableMap.of(); + + // text could have fields + // a read-only collection + @EqualsAndHashCode.Exclude + Map fields = ImmutableMap.of(); + @Override + // Called when building TypeEnvironment and when serializing PPL response public String typeName() { - return jdbcType; + // To avoid breaking changes return `string` for `typeName` call (PPL) and `text` for + // `legacyTypeName` call (SQL). See more: https://github.com/opensearch-project/sql/issues/1296 + if (legacyTypeName().equals("TEXT")) { + return "STRING"; + } + return legacyTypeName(); } @Override + // Called when serializing SQL response public String legacyTypeName() { - return LEGACY_TYPE_NAME_MAPPING.getOrDefault(this, typeName()); + if (mappingType == null) { + return exprCoreType.typeName(); + } + return mappingType.toString().toUpperCase(); + } + + /** + * Clone type object without {@link #properties} - without info about nested object types. + * Note: Should be overriden by all derived classes for proper work. + * @return A cloned object. + */ + protected OpenSearchDataType cloneEmpty() { + var copy = new OpenSearchDataType(); + copy.mappingType = mappingType; + copy.exprCoreType = exprCoreType; + return copy; + } + + /** + * Flattens mapping tree into a single layer list of objects (pairs of name-types actually), + * which don't have nested types. + * See {@link OpenSearchDataTypeTest#traverseAndFlatten() test} for example. + * @param tree A list of `OpenSearchDataType`s - map between field name and its type. + * @return A list of all `OpenSearchDataType`s from given map on the same nesting level (1). + * Nested object names are prefixed by names of their host. + */ + public static Map traverseAndFlatten( + Map tree) { + final Map result = new LinkedHashMap<>(); + BiConsumer, String> visitLevel = new BiConsumer<>() { + @Override + public void accept(Map subtree, String prefix) { + for (var entry : subtree.entrySet()) { + String entryKey = entry.getKey(); + var nextPrefix = prefix.isEmpty() ? entryKey : String.format("%s.%s", prefix, entryKey); + result.put(nextPrefix, entry.getValue().cloneEmpty()); + var nextSubtree = entry.getValue().getProperties(); + if (!nextSubtree.isEmpty()) { + accept(nextSubtree, nextPrefix); + } + } + } + }; + visitLevel.accept(tree, ""); + return result; + } + + /** + * Resolve type of identified from parsed mapping tree. + * @param tree Parsed mapping tree (not flattened). + * @param id An identifier. + * @return Resolved OpenSearchDataType or null if not found. + */ + public static OpenSearchDataType resolve(Map tree, String id) { + for (var item : tree.entrySet()) { + if (item.getKey().equals(id)) { + return item.getValue(); + } + OpenSearchDataType result = resolve(item.getValue().getProperties(), id); + if (result != null) { + return result; + } + } + return null; } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchGeoPointType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchGeoPointType.java new file mode 100644 index 0000000000..c2428a59a8 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchGeoPointType.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.opensearch.data.type; + +import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; + +import lombok.EqualsAndHashCode; + +/** + * The type of a geo_point value. See + * doc + */ +@EqualsAndHashCode(callSuper = false) +public class OpenSearchGeoPointType extends OpenSearchDataType { + + private static final OpenSearchGeoPointType instance = new OpenSearchGeoPointType(); + + private OpenSearchGeoPointType() { + super(MappingType.GeoPoint); + exprCoreType = UNKNOWN; + } + + public static OpenSearchGeoPointType of() { + return OpenSearchGeoPointType.instance; + } + + @Override + protected OpenSearchDataType cloneEmpty() { + return instance; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchIpType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchIpType.java new file mode 100644 index 0000000000..fccafc6caf --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchIpType.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.opensearch.data.type; + +import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; + +import lombok.EqualsAndHashCode; + +/** + * The type of an ip value. See + * doc + */ +@EqualsAndHashCode(callSuper = false) +public class OpenSearchIpType extends OpenSearchDataType { + + private static final OpenSearchIpType instance = new OpenSearchIpType(); + + private OpenSearchIpType() { + super(MappingType.Ip); + exprCoreType = UNKNOWN; + } + + public static OpenSearchIpType of() { + return OpenSearchIpType.instance; + } + + @Override + protected OpenSearchDataType cloneEmpty() { + return instance; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java new file mode 100644 index 0000000000..1098662e65 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java @@ -0,0 +1,76 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.data.type; + +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import lombok.EqualsAndHashCode; +import org.opensearch.sql.data.type.ExprType; + +/** + * The type of a text value. See + * doc + */ +@EqualsAndHashCode(callSuper = false) +public class OpenSearchTextType extends OpenSearchDataType { + + private static final OpenSearchTextType instance = new OpenSearchTextType(); + + private OpenSearchTextType() { + super(MappingType.Text); + exprCoreType = UNKNOWN; + } + + /** + * Create a Text type which has fields. + * @param fields Fields to set for the new type. + * @return A new type object. + */ + public static OpenSearchTextType of(Map fields) { + var res = new OpenSearchTextType(); + res.fields = ImmutableMap.copyOf(fields); + return res; + } + + public static OpenSearchTextType of() { + return OpenSearchTextType.instance; + } + + @Override + public List getParent() { + return List.of(STRING); + } + + @Override + public boolean shouldCast(ExprType other) { + return false; + } + + public Map getFields() { + return fields; + } + + @Override + protected OpenSearchDataType cloneEmpty() { + return OpenSearchTextType.of(fields); + } + + /** + * Text field doesn't have doc value (exception thrown even when you call "get") + * Limitation: assume inner field name is always "keyword". + */ + public static String convertTextToKeyword(String fieldName, ExprType fieldType) { + if (fieldType instanceof OpenSearchTextType + && ((OpenSearchTextType) fieldType).getFields().size() > 0) { + return fieldName + ".keyword"; + } + return fieldName; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprBinaryValue.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprBinaryValue.java index 05a1aad20b..e418832117 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprBinaryValue.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprBinaryValue.java @@ -10,7 +10,8 @@ import org.opensearch.sql.data.model.AbstractExprValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; + /** * OpenSearch BinaryValue. @@ -41,6 +42,6 @@ public Object value() { @Override public ExprType type() { - return OpenSearchDataType.OPENSEARCH_BINARY; + return OpenSearchBinaryType.of(); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprGeoPointValue.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprGeoPointValue.java index ff235cc330..72f7f4a4f2 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprGeoPointValue.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprGeoPointValue.java @@ -6,13 +6,12 @@ package org.opensearch.sql.opensearch.data.value; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_GEO_POINT; - import java.util.Objects; import lombok.Data; import org.opensearch.sql.data.model.AbstractExprValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchGeoPointType; /** * OpenSearch GeoPointValue. @@ -33,7 +32,7 @@ public Object value() { @Override public ExprType type() { - return OPENSEARCH_GEO_POINT; + return OpenSearchGeoPointType.of(); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValue.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValue.java index 8b4021833e..a17deb7e45 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValue.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValue.java @@ -6,13 +6,12 @@ package org.opensearch.sql.opensearch.data.value; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_IP; - import java.util.Objects; import lombok.RequiredArgsConstructor; import org.opensearch.sql.data.model.AbstractExprValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchIpType; /** * OpenSearch IP ExprValue. @@ -30,7 +29,7 @@ public Object value() { @Override public ExprType type() { - return OPENSEARCH_IP; + return OpenSearchIpType.of(); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextKeywordValue.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextKeywordValue.java deleted file mode 100644 index f7dafba23b..0000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextKeywordValue.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.opensearch.data.value; - -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; - -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.type.ExprType; - -/** - * Expression Text Keyword Value, it is an extension of the ExprValue by Elasticsearch. - * This mostly represents a multi-field in OpenSearch which has a text field and a - * keyword field inside to preserve the original text. - */ -public class OpenSearchExprTextKeywordValue extends ExprStringValue { - - public OpenSearchExprTextKeywordValue(String value) { - super(value); - } - - @Override - public ExprType type() { - return OPENSEARCH_TEXT_KEYWORD; - } - -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValue.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValue.java index 93681d6ded..d093588168 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValue.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValue.java @@ -6,13 +6,12 @@ package org.opensearch.sql.opensearch.data.value; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT; - import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; /** - * Expression Text Value, it is a extension of the ExprValue by Elasticsearch. + * Expression Text Value, it is a extension of the ExprValue by OpenSearch. */ public class OpenSearchExprTextValue extends ExprStringValue { public OpenSearchExprTextValue(String value) { @@ -21,6 +20,6 @@ public OpenSearchExprTextValue(String value) { @Override public ExprType type() { - return OPENSEARCH_TEXT; + return OpenSearchTextType.of(); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 2536121e91..034f9227ee 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -6,25 +6,12 @@ package org.opensearch.sql.opensearch.data.value; -import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; -import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; -import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; -import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; -import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.LONG; -import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_BINARY; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_GEO_POINT; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_IP; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER; import com.fasterxml.jackson.core.JsonProcessingException; @@ -58,6 +45,7 @@ import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.utils.Content; import org.opensearch.sql.opensearch.data.utils.ObjectContent; import org.opensearch.sql.opensearch.data.utils.OpenSearchJsonContent; @@ -70,8 +58,22 @@ public class OpenSearchExprValueFactory { /** * The Mapping of Field and ExprType. */ - @Setter - private Map typeMapping; + private final Map typeMapping; + + /** + * Extend existing mapping by new data without overwrite. + * Called from aggregation only {@link AggregationQueryBuilder#buildTypeMapping}. + * @param typeMapping A data type mapping produced by aggregation. + */ + public void extendTypeMapping(Map typeMapping) { + for (var field : typeMapping.keySet()) { + // Prevent overwriting, because aggregation engine may be not aware + // of all niceties of all types. + if (!this.typeMapping.containsKey(field)) { + this.typeMapping.put(field, typeMapping.get(field)); + } + } + } @Getter @Setter @@ -83,39 +85,56 @@ public class OpenSearchExprValueFactory { private final Map> typeActionMap = new ImmutableMap.Builder>() - .put(INTEGER, c -> new ExprIntegerValue(c.intValue())) - .put(LONG, c -> new ExprLongValue(c.longValue())) - .put(SHORT, c -> new ExprShortValue(c.shortValue())) - .put(BYTE, c -> new ExprByteValue(c.byteValue())) - .put(FLOAT, c -> new ExprFloatValue(c.floatValue())) - .put(DOUBLE, c -> new ExprDoubleValue(c.doubleValue())) - .put(STRING, c -> new ExprStringValue(c.stringValue())) - .put(BOOLEAN, c -> ExprBooleanValue.of(c.booleanValue())) - .put(TIMESTAMP, this::parseTimestamp) - .put(DATE, c -> new ExprDateValue(parseTimestamp(c).dateValue().toString())) - .put(TIME, c -> new ExprTimeValue(parseTimestamp(c).timeValue().toString())) - .put(DATETIME, c -> new ExprDatetimeValue(parseTimestamp(c).datetimeValue())) - .put(OPENSEARCH_TEXT, c -> new OpenSearchExprTextValue(c.stringValue())) - .put(OPENSEARCH_TEXT_KEYWORD, c -> new OpenSearchExprTextKeywordValue(c.stringValue())) - .put(OPENSEARCH_IP, c -> new OpenSearchExprIpValue(c.stringValue())) - .put(OPENSEARCH_GEO_POINT, c -> new OpenSearchExprGeoPointValue(c.geoValue().getLeft(), - c.geoValue().getRight())) - .put(OPENSEARCH_BINARY, c -> new OpenSearchExprBinaryValue(c.stringValue())) + .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Integer), + c -> new ExprIntegerValue(c.intValue())) + .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Long), + c -> new ExprLongValue(c.longValue())) + .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Short), + c -> new ExprShortValue(c.shortValue())) + .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Byte), + c -> new ExprByteValue(c.byteValue())) + .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Float), + c -> new ExprFloatValue(c.floatValue())) + .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Double), + c -> new ExprDoubleValue(c.doubleValue())) + .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Text), + c -> new OpenSearchExprTextValue(c.stringValue())) + .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword), + c -> new ExprStringValue(c.stringValue())) + .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Boolean), + c -> ExprBooleanValue.of(c.booleanValue())) + .put(OpenSearchDataType.of(TIMESTAMP), this::parseTimestamp) + .put(OpenSearchDataType.of(DATE), + c -> new ExprDateValue(parseTimestamp(c).dateValue().toString())) + .put(OpenSearchDataType.of(TIME), + c -> new ExprTimeValue(parseTimestamp(c).timeValue().toString())) + .put(OpenSearchDataType.of(DATETIME), + c -> new ExprDatetimeValue(parseTimestamp(c).datetimeValue())) + .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Ip), + c -> new OpenSearchExprIpValue(c.stringValue())) + .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint), + c -> new OpenSearchExprGeoPointValue(c.geoValue().getLeft(), + c.geoValue().getRight())) + .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Binary), + c -> new OpenSearchExprBinaryValue(c.stringValue())) .build(); /** * Constructor of OpenSearchExprValueFactory. */ - public OpenSearchExprValueFactory( - Map typeMapping) { - this.typeMapping = typeMapping; + public OpenSearchExprValueFactory(Map typeMapping) { + this.typeMapping = OpenSearchDataType.traverseAndFlatten(typeMapping); } /** - * The struct construction has the following assumption. 1. The field has OpenSearch Object - * data type. https://www.elastic.co/guide/en/elasticsearch/reference/current/object.html 2. The - * deeper field is flattened in the typeMapping. e.g. {"employ", "STRUCT"} {"employ.id", - * "INTEGER"} {"employ.state", "STRING"} + * The struct construction has the following assumption: + * 1. The field has OpenSearch Object data type. + * See + * docs + * 2. The deeper field is flattened in the typeMapping. e.g. + * { "employ", "STRUCT" } + * { "employ.id", "INTEGER" } + * { "employ.state", "STRING" } */ public ExprValue construct(String jsonString) { try { @@ -145,9 +164,10 @@ private ExprValue parse(Content content, String field, Optional fieldT } ExprType type = fieldType.get(); - if (type == STRUCT) { + if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Object)) + || type == STRUCT) { return parseStruct(content, field); - } else if (type == ARRAY) { + } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested))) { return parseArray(content, field); } else { if (typeActionMap.containsKey(type)) { @@ -171,8 +191,8 @@ private Optional type(String field) { /** * Only default strict_date_optional_time||epoch_millis is supported, * strict_date_optional_time_nanos||epoch_millis if field is date_nanos. - * https://www.elastic.co/guide/en/elasticsearch/reference/current/date.html - * https://www.elastic.co/guide/en/elasticsearch/reference/current/date_nanos.html + * + * docs * The customized date_format is not supported. */ private ExprValue constructTimestamp(String value) { @@ -208,9 +228,9 @@ private ExprValue parseStruct(Content content, String prefix) { } /** - * Todo. ARRAY is not support now. In Elasticsearch, there is no dedicated array data type. - * https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html. The similar data - * type is nested, but it can only allow a list of objects. + * Todo. ARRAY is not completely supported now. In OpenSearch, there is no dedicated array type. + * docs + * The similar data type is nested, but it can only allow a list of objects. */ private ExprValue parseArray(Content content, String prefix) { List result = new ArrayList<>(); @@ -218,7 +238,7 @@ private ExprValue parseArray(Content content, String prefix) { // ExprCoreType.ARRAY does not indicate inner elements type. OpenSearch nested will be an // array of structs, otherwise parseArray currently only supports array of strings. if (v.isString()) { - result.add(parse(v, prefix, Optional.of(STRING))); + result.add(parse(v, prefix, Optional.of(OpenSearchDataType.of(STRING)))); } else { result.add(parse(v, prefix, Optional.of(STRUCT))); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/mapping/IndexMapping.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/mapping/IndexMapping.java index 64bfcc9972..4fdcf0c637 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/mapping/IndexMapping.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/mapping/IndexMapping.java @@ -6,15 +6,15 @@ package org.opensearch.sql.opensearch.mapping; -import static java.util.Collections.emptyMap; - -import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; -import java.util.function.BiConsumer; -import java.util.function.Function; import java.util.stream.Collectors; +import lombok.Getter; import lombok.ToString; +import org.apache.commons.lang3.EnumUtils; import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; /** * OpenSearch index mapping. Because there is no specific behavior for different field types, @@ -24,14 +24,13 @@ public class IndexMapping { /** Field mappings from field name to field type in OpenSearch date type system. */ - private final Map fieldMappings; - - public IndexMapping(Map fieldMappings) { - this.fieldMappings = fieldMappings; - } + @Getter + private final Map fieldMappings; + @SuppressWarnings("unchecked") public IndexMapping(MappingMetadata metaData) { - this.fieldMappings = flatMappings(metaData.getSourceAsMap()); + this.fieldMappings = parseMapping((Map) metaData.getSourceAsMap() + .getOrDefault("properties", null)); } /** @@ -43,62 +42,27 @@ public int size() { return fieldMappings.size(); } - /** - * Return field type by its name. - * - * @param fieldName field name - * @return field type in string. Or null if not exist. - */ - public String getFieldType(String fieldName) { - return fieldMappings.get(fieldName); - } - - /** - * Get all field types and transform raw string type to expected type. - * - * @param transform transform function to transform field type in string to another type - * @param expected field type class - * @return mapping from field name to field type - */ - public Map getAllFieldTypes(Function transform) { - return fieldMappings.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> transform.apply(e.getValue()))); - } - @SuppressWarnings("unchecked") - private Map flatMappings(Map indexMapping) { - ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); - - flatMappings( - ((Map) indexMapping.getOrDefault("properties", emptyMap())), - "", - builder::put); - return builder.build(); + private Map parseMapping(Map indexMapping) { + Map result = new LinkedHashMap<>(); + if (indexMapping != null) { + indexMapping.forEach((k, v) -> { + var innerMap = (Map)v; + // TODO: confirm that only `object` mappings can omit `type` field. + var type = ((String) innerMap.getOrDefault("type", "object")).replace("_", ""); + if (!EnumUtils.isValidEnumIgnoreCase(OpenSearchDataType.MappingType.class, type)) { + // unknown type, e.g. `alias` + // TODO resolve alias reference + return; + } + // TODO read formats for date type + result.put(k, OpenSearchDataType.of( + EnumUtils.getEnumIgnoreCase(OpenSearchDataType.MappingType.class, type), + parseMapping((Map) innerMap.getOrDefault("properties", null)), + parseMapping((Map) innerMap.getOrDefault("fields", null)) + )); + }); + } + return result; } - - @SuppressWarnings("unchecked") - private void flatMappings( - Map mappings, String path, BiConsumer func) { - mappings.forEach( - (fieldName, mappingObject) -> { - Map mapping = (Map) mappingObject; - String fullFieldName = path.isEmpty() ? fieldName : path + "." + fieldName; - - if (isMultiField(mapping)) { - func.accept(fullFieldName, "text_keyword"); - } else { - String type = (String) mapping.getOrDefault("type", "object"); - func.accept(fullFieldName, type); - } - - if (mapping.containsKey("properties")) { // Nested field - flatMappings((Map) mapping.get("properties"), fullFieldName, func); - } - }); - } - - private boolean isMultiField(Map mapping) { - return mapping.containsKey("fields"); - } - } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java index 439a970a4f..97aeee3747 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java @@ -32,9 +32,9 @@ import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.StringUtils; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; @@ -222,8 +222,8 @@ public void pushDownProjects(Set projects) { sourceBuilder.fetchSource(projectsSet.toArray(new String[0]), new String[0]); } - public void pushTypeMapping(Map typeMapping) { - exprValueFactory.setTypeMapping(typeMapping); + public void pushTypeMapping(Map typeMapping) { + exprValueFactory.extendTypeMapping(typeMapping); } private boolean isBoolFilterQuery(QueryBuilder current) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java index 50402fc75b..22ed8c2ffe 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java @@ -15,11 +15,8 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.mapping.IndexMapping; @@ -68,7 +65,8 @@ public List search() { List results = new ArrayList<>(); Map meta = client.meta(); int pos = 0; - for (Map.Entry entry : getFieldTypes().entrySet()) { + for (Map.Entry entry + : OpenSearchDataType.traverseAndFlatten(getFieldTypes()).entrySet()) { results.add( row(entry.getKey(), entry.getValue().legacyTypeName().toLowerCase(), pos++, clusterName(meta))); @@ -81,14 +79,12 @@ public List search() { * * @return mapping of field and type. */ - public Map getFieldTypes() { - Map fieldTypes = new HashMap<>(); + // TODO possible collision if two indices have fields with the same name and different mappings + public Map getFieldTypes() { + Map fieldTypes = new HashMap<>(); Map indexMappings = client.getIndexMappings(indexName.getIndexNames()); for (IndexMapping indexMapping : indexMappings.values()) { - fieldTypes - .putAll(indexMapping.getAllFieldTypes(this::transformESTypeToExprType).entrySet().stream() - .filter(entry -> !ExprCoreType.UNKNOWN.equals(entry.getValue())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + fieldTypes.putAll(indexMapping.getFieldMappings()); } return fieldTypes; } @@ -103,10 +99,6 @@ public Integer getMaxResultWindow() { .values().stream().min(Integer::compare).get(); } - private ExprType transformESTypeToExprType(String openSearchType) { - return OpenSearchDataType.getExprType(openSearchType); - } - private ExprTupleValue row(String fieldName, String fieldType, int position, String clusterName) { LinkedHashMap valueMap = new LinkedHashMap<>(); valueMap.put("TABLE_CAT", stringValue(clusterName)); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index c694769b89..9ed8adf3ee 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -8,6 +8,7 @@ import com.google.common.annotations.VisibleForTesting; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import lombok.RequiredArgsConstructor; import org.opensearch.sql.common.setting.Settings; @@ -46,6 +47,11 @@ public class OpenSearchIndex implements Table { /** * The cached mapping of field and type in index. */ + private Map cachedFieldOpenSearchTypes = null; + + /** + * The cached ExprType of fields. + */ private Map cachedFieldTypes = null; /** @@ -74,7 +80,7 @@ public void create(Map schema) { mappings.put("properties", properties); for (Map.Entry colType : schema.entrySet()) { - properties.put(colType.getKey(), OpenSearchDataType.getOpenSearchType(colType.getValue())); + properties.put(colType.getKey(), colType.getValue().legacyTypeName().toLowerCase()); } client.createIndex(indexName.toString(), mappings); } @@ -84,14 +90,39 @@ public void create(Map schema) { * Need to either handle field name conflicts * or lazy evaluate when query engine pulls field type. */ + /** + * Get simplified parsed mapping info. Unlike {@link #getFieldOpenSearchTypes()} + * it returns a flattened map. + * @return A map between field names and matching `ExprCoreType`s. + */ @Override public Map getFieldTypes() { + if (cachedFieldOpenSearchTypes == null) { + cachedFieldOpenSearchTypes = new OpenSearchDescribeIndexRequest(client, indexName) + .getFieldTypes(); + } if (cachedFieldTypes == null) { - cachedFieldTypes = new OpenSearchDescribeIndexRequest(client, indexName).getFieldTypes(); + cachedFieldTypes = OpenSearchDataType.traverseAndFlatten(cachedFieldOpenSearchTypes) + .entrySet().stream().collect( + LinkedHashMap::new, + (map, item) -> map.put(item.getKey(), item.getValue().getExprType()), + Map::putAll); } return cachedFieldTypes; } + /** + * Get parsed mapping info. + * @return A complete map between field names and their types. + */ + public Map getFieldOpenSearchTypes() { + if (cachedFieldOpenSearchTypes == null) { + cachedFieldOpenSearchTypes = new OpenSearchDescribeIndexRequest(client, indexName) + .getFieldTypes(); + } + return cachedFieldOpenSearchTypes; + } + /** * Get the max result window setting of the table. */ @@ -121,7 +152,7 @@ public LogicalPlan optimize(LogicalPlan plan) { @Override public TableScanBuilder createScanBuilder() { OpenSearchIndexScan indexScan = new OpenSearchIndexScan(client, settings, indexName, - getMaxResultWindow(), new OpenSearchExprValueFactory(getFieldTypes())); + getMaxResultWindow(), new OpenSearchExprValueFactory(getFieldOpenSearchTypes())); return new OpenSearchIndexScanBuilder(indexScan); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/ScriptUtils.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/ScriptUtils.java deleted file mode 100644 index 98a599dfdf..0000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/ScriptUtils.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.opensearch.storage.script; - -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; - -import lombok.experimental.UtilityClass; -import org.opensearch.sql.data.type.ExprType; - -/** - * Script Utils. - */ -@UtilityClass -public class ScriptUtils { - - /** - * Text field doesn't have doc value (exception thrown even when you call "get") - * Limitation: assume inner field name is always "keyword". - */ - public static String convertTextToKeyword(String fieldName, ExprType fieldType) { - if (fieldType == OPENSEARCH_TEXT_KEYWORD) { - return fieldName + ".keyword"; - } - return fieldName; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilder.java index ae3239eea0..1efa5b65d5 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilder.java @@ -24,12 +24,14 @@ import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.ast.tree.Sort; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionNodeVisitor; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.NamedAggregator; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; import org.opensearch.sql.opensearch.response.agg.MetricParser; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; @@ -104,14 +106,16 @@ public AggregationQueryBuilder( } /** - * Build ElasticsearchExprValueFactory. + * Build mapping for OpenSearchExprValueFactory. */ - public Map buildTypeMapping( + public Map buildTypeMapping( List namedAggregatorList, List groupByList) { - ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); - namedAggregatorList.forEach(agg -> builder.put(agg.getName(), agg.type())); - groupByList.forEach(group -> builder.put(group.getNameOrAlias(), group.type())); + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + namedAggregatorList.forEach(agg -> builder.put(agg.getName(), + OpenSearchDataType.of(agg.type()))); + groupByList.forEach(group -> builder.put(group.getNameOrAlias(), + OpenSearchDataType.of(group.type()))); return builder.build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/AggregationBuilderHelper.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/AggregationBuilderHelper.java index 001d7af970..156b565976 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/AggregationBuilderHelper.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/AggregationBuilderHelper.java @@ -17,7 +17,7 @@ import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.ReferenceExpression; -import org.opensearch.sql.opensearch.storage.script.ScriptUtils; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; /** @@ -38,7 +38,8 @@ public T build(Expression expression, Function fieldBuilder, Function scriptBuilder) { if (expression instanceof ReferenceExpression) { String fieldName = ((ReferenceExpression) expression).getAttr(); - return fieldBuilder.apply(ScriptUtils.convertTextToKeyword(fieldName, expression.type())); + return fieldBuilder.apply( + OpenSearchTextType.convertTextToKeyword(fieldName, expression.type())); } else if (expression instanceof FunctionExpression || expression instanceof LiteralExpression) { return scriptBuilder.apply(new Script( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java index 9399c38e46..b327b73b86 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java @@ -28,8 +28,9 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.parse.ParseExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; -import org.opensearch.sql.opensearch.storage.script.ScriptUtils; /** * Expression script executor that executes the expression on each document @@ -105,10 +106,8 @@ public Object visitParse(ParseExpression node, Set context) } private OpenSearchExprValueFactory buildValueFactory(Set fields) { - Map typeEnv = fields.stream() - .collect(toMap( - ReferenceExpression::getAttr, - ReferenceExpression::type)); + Map typeEnv = fields.stream().collect(toMap( + ReferenceExpression::getAttr, e -> OpenSearchDataType.of(e.type()))); return new OpenSearchExprValueFactory(typeEnv); } @@ -128,7 +127,7 @@ private Environment buildValueEnv( private Object getDocValue(ReferenceExpression field, Supplier>> docProvider) { - String fieldName = getDocValueName(field); + String fieldName = OpenSearchTextType.convertTextToKeyword(field.getAttr(), field.type()); ScriptDocValues docValue = docProvider.get().get(fieldName); if (docValue == null || docValue.isEmpty()) { return null; // No way to differentiate null and missing from doc value @@ -141,15 +140,6 @@ private Object getDocValue(ReferenceExpression field, return castNumberToFieldType(value, field.type()); } - /** - * Text field doesn't have doc value (exception thrown even when you call "get") - * Limitation: assume inner field name is always "keyword". - */ - private String getDocValueName(ReferenceExpression field) { - String fieldName = field.getAttr(); - return ScriptUtils.convertTextToKeyword(fieldName, field.type()); - } - /** * DocValue only support long and double so cast to integer and float if needed. * The doc value must be Long and Double for expr type Long/Integer and Double/Float respectively. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LikeQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LikeQuery.java index ad47ba437a..ebb1cd58f0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LikeQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LikeQuery.java @@ -12,13 +12,14 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.script.StringUtils; public class LikeQuery extends LuceneQuery { @Override public QueryBuilder build(FunctionExpression func) { ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0); - String field = convertTextToKeyword(ref.getAttr(), ref.type()); + String field = OpenSearchTextType.convertTextToKeyword(ref.getAttr(), ref.type()); Expression expr = func.getArguments().get(1); ExprValue literalValue = expr.valueOf(); return createBuilder(field, literalValue.stringValue()); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java index 6810ca599c..a8c7ed5e3d 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java @@ -6,8 +6,6 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; - import com.google.common.collect.ImmutableMap; import java.util.Map; import java.util.function.Function; @@ -34,6 +32,7 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; /** * Lucene query abstraction that builds Lucene query from function expression. @@ -223,20 +222,4 @@ protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue l throw new UnsupportedOperationException( "Subclass doesn't implement this and build method either"); } - - /** - * Convert multi-field text field name to its inner keyword field. The limitation and assumption - * is that the keyword field name is always "keyword" which is true by default. - * - * @param fieldName field name - * @param fieldType field type - * @return keyword field name for multi-field, otherwise original field name returned - */ - protected String convertTextToKeyword(String fieldName, ExprType fieldType) { - if (fieldType == OPENSEARCH_TEXT_KEYWORD) { - return fieldName + ".keyword"; - } - return fieldName; - } - } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java index 91aad409f2..c98de1cd84 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java @@ -11,6 +11,7 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; /** * Lucene query that build term query for equality comparison. @@ -19,7 +20,7 @@ public class TermQuery extends LuceneQuery { @Override protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) { - fieldName = convertTextToKeyword(fieldName, fieldType); + fieldName = OpenSearchTextType.convertTextToKeyword(fieldName, fieldType); return QueryBuilders.termQuery(fieldName, value(literal)); } @@ -30,5 +31,4 @@ private Object value(ExprValue literal) { return literal.value(); } } - } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/SortQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/SortQueryBuilder.java index 8fb4eabbd8..ab8f086dff 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/SortQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/SortQueryBuilder.java @@ -15,7 +15,7 @@ import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ReferenceExpression; -import org.opensearch.sql.opensearch.storage.script.ScriptUtils; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; /** * Builder of {@link SortBuilder}. @@ -56,7 +56,8 @@ public SortBuilder build(Expression expression, Sort.SortOption option) { } private FieldSortBuilder fieldBuild(ReferenceExpression ref, Sort.SortOption option) { - return SortBuilders.fieldSort(ScriptUtils.convertTextToKeyword(ref.getAttr(), ref.type())) + return SortBuilders.fieldSort( + OpenSearchTextType.convertTextToKeyword(ref.getAttr(), ref.type())) .order(sortOrderMap.get(option.getSortOrder())) .missing(missingMap.get(option.getNullOrder())); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index ac2f55b5fb..1f987a1ccb 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -6,9 +6,11 @@ package org.opensearch.sql.opensearch.client; +import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Answers.RETURNS_DEEP_STUBS; @@ -20,6 +22,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.opensearch.sql.opensearch.client.OpenSearchClient.META_CLUSTER_NAME; +import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.MappingType; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; @@ -65,6 +68,8 @@ import org.opensearch.sql.data.model.ExprIntegerValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; @@ -156,28 +161,67 @@ void getIndexMappings() throws IOException { mockNodeClientIndicesMappings(indexName, mappings); Map indexMappings = client.getIndexMappings(indexName); - assertEquals(1, indexMappings.size()); - IndexMapping indexMapping = indexMappings.values().iterator().next(); - assertEquals(18, indexMapping.size()); - assertEquals("text", indexMapping.getFieldType("address")); - assertEquals("integer", indexMapping.getFieldType("age")); - assertEquals("double", indexMapping.getFieldType("balance")); - assertEquals("keyword", indexMapping.getFieldType("city")); - assertEquals("date", indexMapping.getFieldType("birthday")); - assertEquals("geo_point", indexMapping.getFieldType("location")); - assertEquals("some_new_es_type_outside_type_system", indexMapping.getFieldType("new_field")); - assertEquals("text", indexMapping.getFieldType("field with spaces")); - assertEquals("text_keyword", indexMapping.getFieldType("employer")); - assertEquals("nested", indexMapping.getFieldType("projects")); - assertEquals("boolean", indexMapping.getFieldType("projects.active")); - assertEquals("date", indexMapping.getFieldType("projects.release")); - assertEquals("nested", indexMapping.getFieldType("projects.members")); - assertEquals("text", indexMapping.getFieldType("projects.members.name")); - assertEquals("object", indexMapping.getFieldType("manager")); - assertEquals("text_keyword", indexMapping.getFieldType("manager.name")); - assertEquals("keyword", indexMapping.getFieldType("manager.address")); - assertEquals("long", indexMapping.getFieldType("manager.salary")); + var mapping = indexMappings.values().iterator().next().getFieldMappings(); + var parsedTypes = OpenSearchDataType.traverseAndFlatten(mapping); + assertAll( + () -> assertEquals(1, indexMappings.size()), + // 10 types extended to 17 after flattening + () -> assertEquals(10, mapping.size()), + () -> assertEquals(17, parsedTypes.size()), + () -> assertEquals("TEXT", mapping.get("address").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Text), + parsedTypes.get("address")), + () -> assertEquals("INTEGER", mapping.get("age").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Integer), + parsedTypes.get("age")), + () -> assertEquals("DOUBLE", mapping.get("balance").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Double), + parsedTypes.get("balance")), + () -> assertEquals("KEYWORD", mapping.get("city").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Keyword), + parsedTypes.get("city")), + () -> assertEquals("DATE", mapping.get("birthday").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Date), + parsedTypes.get("birthday")), + () -> assertEquals("GEO_POINT", mapping.get("location").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.GeoPoint), + parsedTypes.get("location")), + // unknown type isn't parsed and ignored + () -> assertFalse(mapping.containsKey("new_field")), + () -> assertNull(parsedTypes.get("new_field")), + () -> assertEquals("TEXT", mapping.get("field with spaces").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Text), + parsedTypes.get("field with spaces")), + () -> assertEquals("TEXT", mapping.get("employer").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Text), + parsedTypes.get("employer")), + // `employer` is a `text` with `fields` + () -> assertTrue(((OpenSearchTextType)parsedTypes.get("employer")).getFields().size() > 0), + () -> assertEquals("NESTED", mapping.get("projects").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Nested), + parsedTypes.get("projects")), + () -> assertEquals(OpenSearchTextType.of(MappingType.Boolean), + parsedTypes.get("projects.active")), + () -> assertEquals(OpenSearchTextType.of(MappingType.Date), + parsedTypes.get("projects.release")), + () -> assertEquals(OpenSearchTextType.of(MappingType.Nested), + parsedTypes.get("projects.members")), + () -> assertEquals(OpenSearchTextType.of(MappingType.Text), + parsedTypes.get("projects.members.name")), + () -> assertEquals("OBJECT", mapping.get("manager").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Object), + parsedTypes.get("manager")), + () -> assertEquals(OpenSearchTextType.of(MappingType.Text), + parsedTypes.get("manager.name")), + // `manager.name` is a `text` with `fields` + () -> assertTrue(((OpenSearchTextType)parsedTypes.get("manager.name")) + .getFields().size() > 0), + () -> assertEquals(OpenSearchTextType.of(MappingType.Keyword), + parsedTypes.get("manager.address")), + () -> assertEquals(OpenSearchTextType.of(MappingType.Long), + parsedTypes.get("manager.salary")) + ); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index 25cfd6b35c..eec6005d65 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -6,8 +6,10 @@ package org.opensearch.sql.opensearch.client; +import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Answers.RETURNS_DEEP_STUBS; @@ -17,6 +19,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.opensearch.sql.opensearch.client.OpenSearchClient.META_CLUSTER_NAME; +import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.MappingType; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; @@ -57,6 +60,8 @@ import org.opensearch.sql.data.model.ExprIntegerValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; @@ -147,28 +152,67 @@ void getIndexMappings() throws IOException { .thenReturn(response); Map indexMappings = client.getIndexMappings(indexName); - assertEquals(1, indexMappings.size()); - - IndexMapping indexMapping = indexMappings.values().iterator().next(); - assertEquals(18, indexMapping.size()); - assertEquals("text", indexMapping.getFieldType("address")); - assertEquals("integer", indexMapping.getFieldType("age")); - assertEquals("double", indexMapping.getFieldType("balance")); - assertEquals("keyword", indexMapping.getFieldType("city")); - assertEquals("date", indexMapping.getFieldType("birthday")); - assertEquals("geo_point", indexMapping.getFieldType("location")); - assertEquals("some_new_es_type_outside_type_system", indexMapping.getFieldType("new_field")); - assertEquals("text", indexMapping.getFieldType("field with spaces")); - assertEquals("text_keyword", indexMapping.getFieldType("employer")); - assertEquals("nested", indexMapping.getFieldType("projects")); - assertEquals("boolean", indexMapping.getFieldType("projects.active")); - assertEquals("date", indexMapping.getFieldType("projects.release")); - assertEquals("nested", indexMapping.getFieldType("projects.members")); - assertEquals("text", indexMapping.getFieldType("projects.members.name")); - assertEquals("object", indexMapping.getFieldType("manager")); - assertEquals("text_keyword", indexMapping.getFieldType("manager.name")); - assertEquals("keyword", indexMapping.getFieldType("manager.address")); - assertEquals("long", indexMapping.getFieldType("manager.salary")); + + var mapping = indexMappings.values().iterator().next().getFieldMappings(); + var parsedTypes = OpenSearchDataType.traverseAndFlatten(mapping); + assertAll( + () -> assertEquals(1, indexMappings.size()), + // 10 types extended to 17 after flattening + () -> assertEquals(10, mapping.size()), + () -> assertEquals(17, parsedTypes.size()), + () -> assertEquals("TEXT", mapping.get("address").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Text), + parsedTypes.get("address")), + () -> assertEquals("INTEGER", mapping.get("age").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Integer), + parsedTypes.get("age")), + () -> assertEquals("DOUBLE", mapping.get("balance").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Double), + parsedTypes.get("balance")), + () -> assertEquals("KEYWORD", mapping.get("city").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Keyword), + parsedTypes.get("city")), + () -> assertEquals("DATE", mapping.get("birthday").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Date), + parsedTypes.get("birthday")), + () -> assertEquals("GEO_POINT", mapping.get("location").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.GeoPoint), + parsedTypes.get("location")), + // unknown type isn't parsed and ignored + () -> assertFalse(mapping.containsKey("new_field")), + () -> assertNull(parsedTypes.get("new_field")), + () -> assertEquals("TEXT", mapping.get("field with spaces").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Text), + parsedTypes.get("field with spaces")), + () -> assertEquals("TEXT", mapping.get("employer").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Text), + parsedTypes.get("employer")), + // `employer` is a `text` with `fields` + () -> assertTrue(((OpenSearchTextType)parsedTypes.get("employer")).getFields().size() > 0), + () -> assertEquals("NESTED", mapping.get("projects").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Nested), + parsedTypes.get("projects")), + () -> assertEquals(OpenSearchTextType.of(MappingType.Boolean), + parsedTypes.get("projects.active")), + () -> assertEquals(OpenSearchTextType.of(MappingType.Date), + parsedTypes.get("projects.release")), + () -> assertEquals(OpenSearchTextType.of(MappingType.Nested), + parsedTypes.get("projects.members")), + () -> assertEquals(OpenSearchTextType.of(MappingType.Text), + parsedTypes.get("projects.members.name")), + () -> assertEquals("OBJECT", mapping.get("manager").legacyTypeName()), + () -> assertEquals(OpenSearchTextType.of(MappingType.Object), + parsedTypes.get("manager")), + () -> assertEquals(OpenSearchTextType.of(MappingType.Text), + parsedTypes.get("manager.name")), + // `manager.name` is a `text` with `fields` + () -> assertTrue(((OpenSearchTextType)parsedTypes.get("manager.name")) + .getFields().size() > 0), + () -> assertEquals(OpenSearchTextType.of(MappingType.Keyword), + parsedTypes.get("manager.address")), + () -> assertEquals(OpenSearchTextType.of(MappingType.Long), + parsedTypes.get("manager.salary")) + ); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java index 22ccff9d25..c3a5d13dca 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java @@ -7,38 +7,51 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +import java.util.Map; import java.util.stream.Stream; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.opensearch.data.value.OpenSearchExprBinaryValue; import org.opensearch.sql.opensearch.data.value.OpenSearchExprGeoPointValue; import org.opensearch.sql.opensearch.data.value.OpenSearchExprIpValue; -import org.opensearch.sql.opensearch.data.value.OpenSearchExprTextKeywordValue; import org.opensearch.sql.opensearch.data.value.OpenSearchExprTextValue; public class OpenSearchDataTypeRecognitionTest { - @ParameterizedTest + @ParameterizedTest(name = "{2}") @MethodSource("types") - public void typeof(String expected, ExprValue value) { + public void typeof(String expected, ExprValue value, String testName) { assertEquals(expected, typeofGetValue(value)); } private static Stream types() { - // TODO: OPENSEARCH_ARRAY and new types + // TODO: ARRAY and new types return Stream.of( - Arguments.of("OPENSEARCH_TEXT", new OpenSearchExprTextValue("A")), - Arguments.of("OPENSEARCH_BINARY", new OpenSearchExprBinaryValue("A")), - Arguments.of("OPENSEARCH_IP", new OpenSearchExprIpValue("A")), - Arguments.of("OPENSEARCH_TEXT_KEYWORD", new OpenSearchExprTextKeywordValue("A")), - Arguments.of("OPENSEARCH_GEO_POINT", new OpenSearchExprGeoPointValue(0d, 0d)) + Arguments.of("TEXT", new OpenSearchExprTextValue("A"), "text without fields"), + Arguments.of("BINARY", new OpenSearchExprBinaryValue("A"), "binary"), + Arguments.of("IP", new OpenSearchExprIpValue("A"), "ip"), + Arguments.of("TEXT", new TestTextWithFieldValue("Hello World"), "text with fields"), + Arguments.of("GEO_POINT", new OpenSearchExprGeoPointValue(0d, 0d), "geo point") ); } private String typeofGetValue(ExprValue input) { return DSL.typeof(DSL.literal(input)).valueOf().stringValue(); } + + static class TestTextWithFieldValue extends OpenSearchExprTextValue { + public TestTextWithFieldValue(String value) { + super(value); + } + + @Override + public ExprType type() { + return OpenSearchTextType.of(Map.of("words", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword))); + } + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 57bfcd1ea8..5cd76b1962 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -6,62 +6,390 @@ package org.opensearch.sql.opensearch.data.type; +import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; +import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; +import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.MappingType; +import java.util.Map; +import java.util.stream.Stream; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class OpenSearchDataTypeTest { + + private static final OpenSearchDataType textType = OpenSearchDataType.of(MappingType.Text); + private static final OpenSearchDataType textKeywordType = + OpenSearchTextType.of(Map.of("words", OpenSearchTextType.of(MappingType.Keyword))); + @Test - public void testIsCompatible() { - assertTrue(STRING.isCompatible(OPENSEARCH_TEXT)); - assertFalse(OPENSEARCH_TEXT.isCompatible(STRING)); + public void isCompatible() { + assertTrue(STRING.isCompatible(textType)); + assertFalse(textType.isCompatible(STRING)); - assertTrue(STRING.isCompatible(OPENSEARCH_TEXT_KEYWORD)); - assertTrue(OPENSEARCH_TEXT.isCompatible(OPENSEARCH_TEXT_KEYWORD)); + assertTrue(STRING.isCompatible(textKeywordType)); + assertTrue(textType.isCompatible(textKeywordType)); } + // `typeName` and `legacyTypeName` return different things: + // https://github.com/opensearch-project/sql/issues/1296 @Test - public void testTypeName() { - assertEquals("string", OPENSEARCH_TEXT.typeName()); - assertEquals("string", OPENSEARCH_TEXT_KEYWORD.typeName()); + public void typeName() { + assertEquals("STRING", textType.typeName()); + assertEquals("STRING", textKeywordType.typeName()); + assertEquals("OBJECT", OpenSearchDataType.of(MappingType.Object).typeName()); + assertEquals("DATE", OpenSearchDataType.of(MappingType.Date).typeName()); + assertEquals("DOUBLE", OpenSearchDataType.of(MappingType.Double).typeName()); + assertEquals("KEYWORD", OpenSearchDataType.of(MappingType.Keyword).typeName()); } @Test public void legacyTypeName() { - assertEquals("text", OPENSEARCH_TEXT.legacyTypeName()); - assertEquals("text", OPENSEARCH_TEXT_KEYWORD.legacyTypeName()); + assertEquals("TEXT", textType.legacyTypeName()); + assertEquals("TEXT", textKeywordType.legacyTypeName()); + assertEquals("OBJECT", OpenSearchDataType.of(MappingType.Object).legacyTypeName()); + assertEquals("DATE", OpenSearchDataType.of(MappingType.Date).legacyTypeName()); + assertEquals("DOUBLE", OpenSearchDataType.of(MappingType.Double).legacyTypeName()); + assertEquals("KEYWORD", OpenSearchDataType.of(MappingType.Keyword).legacyTypeName()); + } + + @Test + public void shouldCast() { + assertFalse(textType.shouldCast(STRING)); + assertFalse(textKeywordType.shouldCast(STRING)); + } + + private static Stream getTestDataWithType() { + return Stream.of( + Arguments.of(MappingType.Text, "text", OpenSearchTextType.of()), + Arguments.of(MappingType.Keyword, "keyword", STRING), + Arguments.of(MappingType.Byte, "byte", BYTE), + Arguments.of(MappingType.Short, "short", SHORT), + Arguments.of(MappingType.Integer, "integer", INTEGER), + Arguments.of(MappingType.Long, "long", LONG), + Arguments.of(MappingType.HalfFloat, "half_float", FLOAT), + Arguments.of(MappingType.Float, "float", FLOAT), + Arguments.of(MappingType.ScaledFloat, "scaled_float", DOUBLE), + Arguments.of(MappingType.Double, "double", DOUBLE), + Arguments.of(MappingType.Boolean, "boolean", BOOLEAN), + Arguments.of(MappingType.Date, "date", TIMESTAMP), + Arguments.of(MappingType.Object, "object", STRUCT), + Arguments.of(MappingType.Nested, "nested", ARRAY), + Arguments.of(MappingType.GeoPoint, "geo_point", + OpenSearchGeoPointType.of()), + Arguments.of(MappingType.Binary, "binary", + OpenSearchBinaryType.of()), + Arguments.of(MappingType.Ip, "ip", + OpenSearchIpType.of()) + ); + } + + @ParameterizedTest(name = "{1}") + @MethodSource("getTestDataWithType") + public void of_MappingType(MappingType mappingType, String name, ExprType dataType) { + var type = OpenSearchDataType.of(mappingType); + // For serialization of SQL and PPL different functions are used, and it was designed to return + // different types. No clue why, but it should be fixed in #1296. + var nameForSQL = name.toUpperCase(); + var nameForPPL = name.equals("text") ? "STRING" : name.toUpperCase(); + assertAll( + () -> assertEquals(nameForPPL, type.typeName()), + () -> assertEquals(nameForSQL, type.legacyTypeName()), + () -> assertEquals(dataType, type.getExprType()) + ); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(ExprCoreType.class) + public void of_ExprCoreType(ExprCoreType coreType) { + assumeFalse(coreType == UNKNOWN); + var type = OpenSearchDataType.of(coreType); + assertAll( + () -> assertEquals(coreType.toString(), type.typeName()), + () -> assertEquals(coreType.toString(), type.legacyTypeName()), + () -> assertEquals(coreType, type.getExprType()) + ); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(ExprCoreType.class) + public void of_OpenSearchDataType_from_ExprCoreType(ExprCoreType coreType) { + var type = OpenSearchDataType.of(coreType); + var derivedType = OpenSearchDataType.of(type); + assertEquals(type, derivedType); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(MappingType.class) + public void of_OpenSearchDataType_from_MappingType(OpenSearchDataType.MappingType mappingType) { + assumeFalse(mappingType == MappingType.Invalid); + var type = OpenSearchDataType.of(mappingType); + var derivedType = OpenSearchDataType.of(type); + assertEquals(type, derivedType); + } + + @Test + // All types without `fields` and `properties` are singletones unless cloned. + public void types_but_clones_are_singletones_and_cached() { + var type = OpenSearchDataType.of(MappingType.Object); + var alsoType = OpenSearchDataType.of(MappingType.Object); + var typeWithProperties = OpenSearchDataType.of(MappingType.Object, + Map.of("subfield", OpenSearchDataType.of(INTEGER)), Map.of()); + var typeWithFields = OpenSearchDataType.of(MappingType.Text, + Map.of(), Map.of("subfield", OpenSearchDataType.of(INTEGER))); + + var cloneType = type.cloneEmpty(); + assertAll( + () -> assertSame(type, alsoType), + () -> assertNotSame(type, cloneType), + () -> assertNotSame(type, typeWithProperties), + () -> assertNotSame(type, typeWithFields), + () -> assertNotSame(typeWithProperties, typeWithProperties.cloneEmpty()), + () -> assertNotSame(typeWithFields, typeWithFields.cloneEmpty()), + () -> assertSame(OpenSearchDataType.of(MappingType.Text), + OpenSearchTextType.of()), + () -> assertSame(OpenSearchDataType.of(MappingType.Binary), + OpenSearchBinaryType.of()), + () -> assertSame(OpenSearchDataType.of(MappingType.GeoPoint), + OpenSearchGeoPointType.of()), + () -> assertSame(OpenSearchDataType.of(MappingType.Ip), + OpenSearchIpType.of()), + () -> assertNotSame(OpenSearchTextType.of(), + OpenSearchTextType.of(Map.of("subfield", OpenSearchDataType.of(INTEGER)))), + () -> assertSame(OpenSearchDataType.of(INTEGER), OpenSearchDataType.of(INTEGER)), + () -> assertSame(OpenSearchDataType.of(STRING), OpenSearchDataType.of(STRING)), + () -> assertSame(OpenSearchDataType.of(STRUCT), OpenSearchDataType.of(STRUCT)), + () -> assertNotSame(OpenSearchDataType.of(INTEGER), + OpenSearchDataType.of(INTEGER).cloneEmpty()) + ); + } + + @Test + // Use OpenSearchDataType.of(type, properties, fields) or OpenSearchTextType.of(fields) + // to create a new type object with required parameters. Types are immutable, even clones. + public void fields_and_properties_are_readonly() { + var objectType = OpenSearchDataType.of(MappingType.Object); + var textType = OpenSearchTextType.of(); + var textTypeWithFields = OpenSearchTextType.of( + Map.of("letters", OpenSearchDataType.of(MappingType.Keyword))); + assertAll( + () -> assertThrows(UnsupportedOperationException.class, + () -> objectType.getProperties().put("something", OpenSearchDataType.of(INTEGER))), + () -> assertThrows(UnsupportedOperationException.class, + () -> textType.getFields().put("words", OpenSearchDataType.of(MappingType.Keyword))), + () -> assertThrows(UnsupportedOperationException.class, + () -> textTypeWithFields.getFields().put("words", + OpenSearchDataType.of(MappingType.Keyword))) + ); } @Test - public void testShouldCast() { - assertFalse(OPENSEARCH_TEXT.shouldCast(STRING)); - assertFalse(OPENSEARCH_TEXT_KEYWORD.shouldCast(STRING)); + // Test and type added for coverage only + public void of_null_MappingType() { + assertThrows(IllegalArgumentException.class, () -> OpenSearchDataType.of(MappingType.Invalid)); } @Test - public void testGetExprType() { - assertEquals(OPENSEARCH_TEXT, OpenSearchDataType.getExprType("text")); - assertEquals(FLOAT, OpenSearchDataType.getExprType("float")); - assertEquals(FLOAT, OpenSearchDataType.getExprType("half_float")); - assertEquals(DOUBLE, OpenSearchDataType.getExprType("double")); - assertEquals(DOUBLE, OpenSearchDataType.getExprType("scaled_float")); - assertEquals(TIMESTAMP, OpenSearchDataType.getExprType("date")); - assertEquals(TIMESTAMP, OpenSearchDataType.getExprType("date_nanos")); + // cloneEmpty doesn't clone properties and fields. + // Fields are cloned by OpenSearchTextType::cloneEmpty, because it is used in that type only. + public void cloneEmpty() { + var type = OpenSearchDataType.of(MappingType.Object, + Map.of("val", OpenSearchDataType.of(INTEGER)), + Map.of("words", OpenSearchDataType.of(STRING))); + var clone = type.cloneEmpty(); + var textClone = textKeywordType.cloneEmpty(); + + assertAll( + // can compare because `properties` and `fields` are marked as @EqualsAndHashCode.Exclude + () -> assertEquals(type, clone), + // read private field `fields` + () -> assertTrue( + ((Map) FieldUtils.readField(clone, "fields", true)) + .isEmpty()), + () -> assertTrue(clone.getProperties().isEmpty()), + () -> assertEquals(textKeywordType, textClone), + () -> assertEquals(FieldUtils.readField(textKeywordType, "fields", true), + FieldUtils.readField(textClone, "fields", true)) + ); + } + + // Following structure of nested objects should be flattened + // ===================== + // type + // |- subtype + // | | + // | |- subsubtype + // | | | + // | | |- textWithKeywordType + // | | | |- keyword + // | | | + // | | |- INTEGER + // | | + // | |- GeoPoint + // | |- textWithFieldsType + // | |- words + // | + // |- text + // |- keyword + // ================= + // as + // ================= + // type : Object + // type.subtype : Object + // type.subtype.subsubtype : Object + // type.subtype.subsubtype.textWithKeywordType : Text + // |- keyword : Keyword + // type.subtype.subsubtype.INTEGER : INTEGER + // type.subtype.geo_point : GeoPoint + // type.subtype.textWithFieldsType: Text + // |- words : Keyword + // type.text : Text + // type.keyword : Keyword + // ================== + // Objects are flattened by OpenSearch, but Texts aren't + // TODO Arrays + @Test + public void traverseAndFlatten() { + var flattened = OpenSearchDataType.traverseAndFlatten(getSampleMapping()); + var objectType = OpenSearchDataType.of(MappingType.Object); + assertAll( + () -> assertEquals(9, flattened.size()), + () -> assertTrue(flattened.get("type").getProperties().isEmpty()), + () -> assertTrue(flattened.get("type.subtype").getProperties().isEmpty()), + () -> assertTrue(flattened.get("type.subtype.subsubtype").getProperties().isEmpty()), + + () -> assertEquals(objectType, flattened.get("type")), + () -> assertEquals(objectType, flattened.get("type.subtype")), + () -> assertEquals(objectType, flattened.get("type.subtype.subsubtype")), + + () -> assertEquals(OpenSearchDataType.of(MappingType.Keyword), + flattened.get("type.keyword")), + () -> assertEquals(OpenSearchDataType.of(MappingType.Text), + flattened.get("type.text")), + + () -> assertEquals(OpenSearchGeoPointType.of(), + flattened.get("type.subtype.geo_point")), + () -> assertEquals(OpenSearchTextType.of(), + flattened.get("type.subtype.textWithFieldsType")), + + () -> assertEquals(OpenSearchTextType.of(), + flattened.get("type.subtype.subsubtype.textWithKeywordType")), + () -> assertEquals(OpenSearchDataType.of(INTEGER), + flattened.get("type.subtype.subsubtype.INTEGER")) + ); + } + + @Test + public void resolve() { + var mapping = getSampleMapping(); + + assertAll( + () -> assertNull(OpenSearchDataType.resolve(mapping, "incorrect")), + () -> assertEquals(OpenSearchDataType.of(MappingType.Object), + OpenSearchDataType.resolve(mapping, "type")), + () -> assertEquals(OpenSearchDataType.of(MappingType.Object), + OpenSearchDataType.resolve(mapping, "subtype")), + () -> assertEquals(OpenSearchDataType.of(MappingType.Object), + OpenSearchDataType.resolve(mapping, "subsubtype")), + () -> assertEquals(OpenSearchDataType.of(MappingType.Text), + OpenSearchDataType.resolve(mapping, "textWithKeywordType")), + () -> assertEquals(OpenSearchDataType.of(MappingType.Text), + OpenSearchDataType.resolve(mapping, "textWithFieldsType")), + () -> assertEquals(OpenSearchDataType.of(MappingType.Text), + OpenSearchDataType.resolve(mapping, "text")), + () -> assertEquals(OpenSearchDataType.of(MappingType.Integer), + OpenSearchDataType.resolve(mapping, "INTEGER")), + () -> assertEquals(OpenSearchDataType.of(MappingType.GeoPoint), + OpenSearchDataType.resolve(mapping, "geo_point")), + () -> assertEquals(OpenSearchDataType.of(MappingType.Keyword), + OpenSearchDataType.resolve(mapping, "keyword")) + ); + } + + // type : Object + // type.subtype : Object + // type.subtype.subsubtype : Object + // type.subtype.subsubtype.textWithKeywordType : Text + // |- keyword : Keyword + // type.subtype.subsubtype.INTEGER : INTEGER + // type.subtype.geo_point : GeoPoint + // type.subtype.textWithFieldsType: Text + // |- words : Keyword + // type.text : Text + // type.keyword : Keyword + + @Test + public void text_type_with_fields_ctor() { + var type = OpenSearchTextType.of(Map.of("words", + OpenSearchDataType.of(MappingType.Keyword))); + assertAll( + () -> assertEquals(OpenSearchTextType.of(), type), + () -> assertEquals(1, type.getFields().size()), + () -> assertEquals(OpenSearchDataType.of(MappingType.Keyword), + type.getFields().get("words")) + ); + } + + private Map getSampleMapping() { + var textWithKeywordType = OpenSearchTextType.of(Map.of("keyword", + OpenSearchDataType.of(MappingType.Keyword))); + + var subsubsubtypes = Map.of( + "textWithKeywordType", textWithKeywordType, + "INTEGER", OpenSearchDataType.of(INTEGER)); + + var subsubtypes = Map.of( + "subsubtype", OpenSearchDataType.of(MappingType.Object, + subsubsubtypes, Map.of()), + "textWithFieldsType", OpenSearchDataType.of(MappingType.Text, Map.of(), + Map.of("words", OpenSearchDataType.of(MappingType.Keyword))), + "geo_point", OpenSearchGeoPointType.of()); + + var subtypes = Map.of( + "subtype", OpenSearchDataType.of(MappingType.Object, + subsubtypes, Map.of()), + "keyword", OpenSearchDataType.of(MappingType.Keyword), + "text", OpenSearchDataType.of(MappingType.Text)); + + var type = OpenSearchDataType.of(MappingType.Object, subtypes, Map.of()); + return Map.of("type", type); } @Test - public void testGetOpenSearchType() { - assertEquals("text", OpenSearchDataType.getOpenSearchType(OPENSEARCH_TEXT)); - assertEquals("float", OpenSearchDataType.getOpenSearchType(FLOAT)); - assertEquals("double", OpenSearchDataType.getOpenSearchType(DOUBLE)); - assertEquals("date", OpenSearchDataType.getOpenSearchType(TIMESTAMP)); + public void test_getExprType() { + assertEquals(OpenSearchTextType.of(), + OpenSearchDataType.of(MappingType.Text).getExprType()); + assertEquals(FLOAT, OpenSearchDataType.of(MappingType.Float).getExprType()); + assertEquals(FLOAT, OpenSearchDataType.of(MappingType.HalfFloat).getExprType()); + assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.Double).getExprType()); + assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.ScaledFloat).getExprType()); + assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprType()); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprBinaryValueTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprBinaryValueTest.java index 35f84a6e63..4e7b33f944 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprBinaryValueTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprBinaryValueTest.java @@ -8,9 +8,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_BINARY; import org.junit.jupiter.api.Test; +import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; public class OpenSearchExprBinaryValueTest { @@ -40,7 +40,6 @@ public void value() { public void type() { OpenSearchExprBinaryValue value = new OpenSearchExprBinaryValue("U29tZSBiaW5hcnkgYmxvYg=="); - assertEquals(OPENSEARCH_BINARY, value.type()); + assertEquals(OpenSearchBinaryType.of(), value.type()); } - } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprGeoPointValueTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprGeoPointValueTest.java index 302a2aad96..4edb25aff5 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprGeoPointValueTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprGeoPointValueTest.java @@ -9,14 +9,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_GEO_POINT; import org.junit.jupiter.api.Test; +import org.opensearch.sql.opensearch.data.type.OpenSearchGeoPointType; class OpenSearchExprGeoPointValueTest { - private OpenSearchExprGeoPointValue geoPointValue = new OpenSearchExprGeoPointValue(1.0, - 1.0); + private OpenSearchExprGeoPointValue geoPointValue = new OpenSearchExprGeoPointValue(1.0, 1.0); @Test void value() { @@ -25,18 +24,18 @@ void value() { @Test void type() { - assertEquals(OPENSEARCH_GEO_POINT, geoPointValue.type()); + assertEquals(OpenSearchGeoPointType.of(), geoPointValue.type()); } @Test void compare() { assertEquals(0, geoPointValue.compareTo(new OpenSearchExprGeoPointValue(1.0, 1.0))); + assertEquals(geoPointValue, new OpenSearchExprGeoPointValue(1.0, 1.0)); } @Test void equal() { - assertTrue(geoPointValue.equal(new OpenSearchExprGeoPointValue(1.0, - 1.0))); + assertTrue(geoPointValue.equal(new OpenSearchExprGeoPointValue(1.0, 1.0))); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java index baaceb2641..cda4377c60 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java @@ -9,9 +9,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_IP; import org.junit.jupiter.api.Test; +import org.opensearch.sql.opensearch.data.type.OpenSearchIpType; public class OpenSearchExprIpValueTest { @@ -24,12 +24,13 @@ void value() { @Test void type() { - assertEquals(OPENSEARCH_IP, ipValue.type()); + assertEquals(OpenSearchIpType.of(), ipValue.type()); } @Test void compare() { assertEquals(0, ipValue.compareTo(new OpenSearchExprIpValue("192.168.0.1"))); + assertEquals(ipValue, new OpenSearchExprIpValue("192.168.0.1")); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextKeywordValueTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextKeywordValueTest.java deleted file mode 100644 index ee2bbaf13c..0000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextKeywordValueTest.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.opensearch.data.value; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; - -import org.junit.jupiter.api.Test; - -class OpenSearchExprTextKeywordValueTest { - - @Test - public void testTypeOfExprTextKeywordValue() { - assertEquals(OPENSEARCH_TEXT_KEYWORD, new OpenSearchExprTextKeywordValue("A").type()); - } - -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java index bb7a6b415b..2dfa5de93a 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java @@ -6,14 +6,75 @@ package org.opensearch.sql.opensearch.data.value; +import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import java.util.Map; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class OpenSearchExprTextValueTest { @Test - public void typeOfExprTextValue() { - assertEquals(OPENSEARCH_TEXT, new OpenSearchExprTextValue("A").type()); + public void type_of_ExprTextValue() { + assertEquals(OpenSearchTextType.of(), new OpenSearchExprTextValue("A").type()); + } + + @Test + public void getFields() { + var fields = Map.of( + "f1", OpenSearchDataType.of(OpenSearchDataType.MappingType.Integer), + "f2", OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword), + "f3", OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword)); + assertEquals(fields, OpenSearchTextType.of(fields).getFields()); + } + + @Test + void non_text_types_arent_converted() { + assertAll( + () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", + OpenSearchDataType.of(INTEGER))), + () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", + OpenSearchDataType.of(STRING))), + () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", + OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint))), + () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword))), + () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Integer))), + () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", STRING)), + () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", INTEGER)) + ); + } + + @Test + void non_text_types_with_nested_objects_arent_converted() { + var objectType = OpenSearchDataType.of(OpenSearchDataType.MappingType.Object, + Map.of("subfield", OpenSearchDataType.of(STRING)), Map.of()); + var arrayType = OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested, + Map.of("subfield", OpenSearchDataType.of(STRING)), Map.of()); + assertAll( + () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", objectType)), + () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", arrayType)) + ); + } + + @Test + void text_type_without_fields_isnt_converted() { + assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Text))); + } + + @Test + void text_type_with_fields_is_converted() { + var textWithKeywordType = OpenSearchTextType.of(Map.of("keyword", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword))); + assertEquals("field.keyword", + OpenSearchTextType.convertTextToKeyword("field", textWithKeywordType)); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index 8d5552d6a8..1b9a8b7e65 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -6,6 +6,7 @@ package org.opensearch.sql.opensearch.data.value; +import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -32,11 +33,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_BINARY; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_GEO_POINT; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_IP; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -54,38 +50,41 @@ import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.data.utils.OpenSearchJsonContent; class OpenSearchExprValueFactoryTest { - private static final Map MAPPING = - new ImmutableMap.Builder() - .put("byteV", BYTE) - .put("shortV", SHORT) - .put("intV", INTEGER) - .put("longV", LONG) - .put("floatV", FLOAT) - .put("doubleV", DOUBLE) - .put("stringV", STRING) - .put("dateV", DATE) - .put("datetimeV", DATETIME) - .put("timeV", TIME) - .put("timestampV", TIMESTAMP) - .put("boolV", BOOLEAN) - .put("structV", STRUCT) - .put("structV.id", INTEGER) - .put("structV.state", STRING) - .put("arrayV", ARRAY) - .put("arrayV.info", STRING) - .put("arrayV.author", STRING) - .put("textV", OPENSEARCH_TEXT) - .put("textKeywordV", OPENSEARCH_TEXT_KEYWORD) - .put("ipV", OPENSEARCH_IP) - .put("geoV", OPENSEARCH_GEO_POINT) - .put("binaryV", OPENSEARCH_BINARY) + private static final Map MAPPING = + new ImmutableMap.Builder() + .put("byteV", OpenSearchDataType.of(BYTE)) + .put("shortV", OpenSearchDataType.of(SHORT)) + .put("intV", OpenSearchDataType.of(INTEGER)) + .put("longV", OpenSearchDataType.of(LONG)) + .put("floatV", OpenSearchDataType.of(FLOAT)) + .put("doubleV", OpenSearchDataType.of(DOUBLE)) + .put("stringV", OpenSearchDataType.of(STRING)) + .put("dateV", OpenSearchDataType.of(DATE)) + .put("datetimeV", OpenSearchDataType.of(DATETIME)) + .put("timeV", OpenSearchDataType.of(TIME)) + .put("timestampV", OpenSearchDataType.of(TIMESTAMP)) + .put("boolV", OpenSearchDataType.of(BOOLEAN)) + .put("structV", OpenSearchDataType.of(STRUCT)) + .put("structV.id", OpenSearchDataType.of(INTEGER)) + .put("structV.state", OpenSearchDataType.of(STRING)) + .put("arrayV", OpenSearchDataType.of(ARRAY)) + .put("arrayV.info", OpenSearchDataType.of(STRING)) + .put("arrayV.author", OpenSearchDataType.of(STRING)) + .put("textV", OpenSearchDataType.of(OpenSearchDataType.MappingType.Text)) + .put("textKeywordV", OpenSearchTextType.of(Map.of("words", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword)))) + .put("ipV", OpenSearchDataType.of(OpenSearchDataType.MappingType.Ip)) + .put("geoV", OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint)) + .put("binaryV", OpenSearchDataType.of(OpenSearchDataType.MappingType.Binary)) .build(); - private OpenSearchExprValueFactory exprValueFactory = + + private final OpenSearchExprValueFactory exprValueFactory = new OpenSearchExprValueFactory(MAPPING); @Test @@ -167,9 +166,9 @@ public void constructText() { assertEquals(new OpenSearchExprTextValue("text"), constructFromObject("textV", "text")); - assertEquals(new OpenSearchExprTextKeywordValue("text"), + assertEquals(new OpenSearchExprTextValue("text"), tupleValue("{\"textKeywordV\":\"text\"}").get("textKeywordV")); - assertEquals(new OpenSearchExprTextKeywordValue("text"), + assertEquals(new OpenSearchExprTextValue("text"), constructFromObject("textKeywordV", "text")); } @@ -376,6 +375,29 @@ public void constructUnsupportedTypeThrowException() { exception.getMessage()); } + @Test + // aggregation adds info about new columns to the factory, + // it is accepted without overwriting existing data. + public void factoryMappingsAreExtendableWithoutOverWrite() + throws NoSuchFieldException, IllegalAccessException { + var factory = new OpenSearchExprValueFactory(Map.of("value", OpenSearchDataType.of(INTEGER))); + factory.extendTypeMapping(Map.of( + "value", OpenSearchDataType.of(DOUBLE), + "agg", OpenSearchDataType.of(DATE))); + // extract private field for testing purposes + var field = factory.getClass().getDeclaredField("typeMapping"); + field.setAccessible(true); + @SuppressWarnings("unchecked") + var mapping = (Map)field.get(factory); + assertAll( + () -> assertEquals(2, mapping.size()), + () -> assertTrue(mapping.containsKey("value")), + () -> assertTrue(mapping.containsKey("agg")), + () -> assertEquals(OpenSearchDataType.of(INTEGER), mapping.get("value")), + () -> assertEquals(OpenSearchDataType.of(DATE), mapping.get("agg")) + ); + } + public Map tupleValue(String jsonString) { final ExprValue construct = exprValueFactory.construct(jsonString); return construct.tupleValue(); @@ -385,9 +407,18 @@ private ExprValue constructFromObject(String fieldName, Object value) { return exprValueFactory.construct(fieldName, value); } - @EqualsAndHashCode + @EqualsAndHashCode(callSuper = false) @ToString - private static class TestType implements ExprType { + private static class TestType extends OpenSearchDataType { + + public TestType() { + mappingType = null; + } + + @Override + protected OpenSearchDataType cloneEmpty() { + return this; + } @Override public String typeName() { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/mapping/IndexMappingTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/mapping/IndexMappingTest.java deleted file mode 100644 index 302dbb1d51..0000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/mapping/IndexMappingTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.opensearch.mapping; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.aMapWithSize; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.hasEntry; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - -import com.google.common.collect.ImmutableMap; -import java.util.Map; -import org.junit.jupiter.api.Test; - -class IndexMappingTest { - - @Test - public void getFieldType() { - IndexMapping indexMapping = new IndexMapping(ImmutableMap.of("name", "text")); - assertEquals("text", indexMapping.getFieldType("name")); - assertNull(indexMapping.getFieldType("not_exist")); - } - - @Test - public void getAllFieldTypes() { - IndexMapping indexMapping = new IndexMapping(ImmutableMap.of("name", "text", "age", "int")); - Map fieldTypes = indexMapping.getAllFieldTypes(type -> "our_type"); - assertThat( - fieldTypes, - allOf(aMapWithSize(2), hasEntry("name", "our_type"), hasEntry("age", "our_type"))); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java index 33376ece83..980d68ed80 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java @@ -37,6 +37,7 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; @@ -215,9 +216,9 @@ void testPushDownProject() { @Test void testPushTypeMapping() { - Map typeMapping = Map.of("intA", INTEGER); + Map typeMapping = Map.of("intA", OpenSearchDataType.of(INTEGER)); requestBuilder.pushTypeMapping(typeMapping); - verify(exprValueFactory).setTypeMapping(typeMapping); + verify(exprValueFactory).extendTypeMapping(typeMapping); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequestTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequestTest.java index f387219814..111316a7ed 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequestTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequestTest.java @@ -12,7 +12,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.when; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; import com.google.common.collect.ImmutableMap; import java.util.List; @@ -22,8 +21,8 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.mapping.IndexMapping; @ExtendWith(MockitoExtension.class) @@ -32,16 +31,14 @@ class OpenSearchDescribeIndexRequestTest { @Mock private OpenSearchClient client; + @Mock + private IndexMapping mapping; + @Test void testSearch() { - when(client.getIndexMappings("index")) - .thenReturn( - ImmutableMap.of( - "test", - new IndexMapping( - ImmutableMap.builder() - .put("name", "keyword") - .build()))); + when(mapping.getFieldMappings()).thenReturn( + Map.of("name", OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword))); + when(client.getIndexMappings("index")).thenReturn(ImmutableMap.of("test", mapping)); final List results = new OpenSearchDescribeIndexRequest(client, "index").search(); assertEquals(1, results.size()); @@ -57,22 +54,4 @@ void testToString() { assertEquals("OpenSearchDescribeIndexRequest{indexName='index'}", new OpenSearchDescribeIndexRequest(client, "index").toString()); } - - @Test - void filterOutUnknownType() { - when(client.getIndexMappings("index")) - .thenReturn( - ImmutableMap.of( - "test", - new IndexMapping( - ImmutableMap.builder() - .put("name", "keyword") - .put("@timestamp", "alias") - .build()))); - - final Map fieldTypes = - new OpenSearchDescribeIndexRequest(client, "index").getFieldTypes(); - assertEquals(1, fieldTypes.size()); - assertThat(fieldTypes, hasEntry("name", STRING)); - } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScanTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScanTest.java index 9a606750a3..8aec6a7d13 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScanTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexScanTest.java @@ -17,7 +17,6 @@ import static org.opensearch.search.sort.SortOrder.ASC; import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import com.google.common.collect.ImmutableMap; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -40,6 +39,7 @@ import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.request.OpenSearchQueryRequest; import org.opensearch.sql.opensearch.request.OpenSearchRequest; @@ -55,7 +55,8 @@ class OpenSearchIndexScanTest { private Settings settings; private OpenSearchExprValueFactory exprValueFactory = new OpenSearchExprValueFactory( - ImmutableMap.of("name", STRING, "department", STRING)); + Map.of("name", OpenSearchDataType.of(STRING), + "department", OpenSearchDataType.of(STRING))); @BeforeEach void setup() { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index 74c18f7c3d..8d4dad48a9 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -13,6 +13,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -20,10 +21,9 @@ import static org.opensearch.sql.expression.DSL.literal; import static org.opensearch.sql.expression.DSL.named; import static org.opensearch.sql.expression.DSL.ref; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; +import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.MappingType; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.eval; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.project; -import static org.opensearch.sql.planner.logical.LogicalPlanDSL.relation; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.remove; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.rename; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.sort; @@ -33,6 +33,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.BeforeEach; @@ -52,6 +53,7 @@ import org.opensearch.sql.expression.aggregation.NamedAggregator; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.planner.logical.LogicalPlan; @@ -76,6 +78,9 @@ class OpenSearchIndexTest { @Mock private Table table; + @Mock + private IndexMapping mapping; + private OpenSearchIndex index; @BeforeEach @@ -95,38 +100,38 @@ void createIndex() { Map mappings = ImmutableMap.of( "properties", ImmutableMap.of( - "name", "text_keyword", + "name", "text", "age", "integer")); doNothing().when(client).createIndex(indexName, mappings); Map schema = new HashMap<>(); - schema.put("name", OPENSEARCH_TEXT_KEYWORD); + schema.put("name", OpenSearchTextType.of(Map.of("keyword", + OpenSearchDataType.of(MappingType.Keyword)))); schema.put("age", INTEGER); index.create(schema); } @Test void getFieldTypes() { - when(client.getIndexMappings("test")) - .thenReturn( - ImmutableMap.of( - "test", - new IndexMapping( - ImmutableMap.builder() - .put("name", "keyword") - .put("address", "text") - .put("age", "integer") - .put("account_number", "long") - .put("balance1", "float") - .put("balance2", "double") - .put("gender", "boolean") - .put("family", "nested") - .put("employer", "object") - .put("birthday", "date") - .put("id1", "byte") - .put("id2", "short") - .put("blob", "binary") - .build()))); + when(mapping.getFieldMappings()).thenReturn( + ImmutableMap.builder() + .put("name", MappingType.Keyword) + .put("address", MappingType.Text) + .put("age", MappingType.Integer) + .put("account_number", MappingType.Long) + .put("balance1", MappingType.Float) + .put("balance2", MappingType.Double) + .put("gender", MappingType.Boolean) + .put("family", MappingType.Nested) + .put("employer", MappingType.Object) + .put("birthday", MappingType.Date) + .put("id1", MappingType.Byte) + .put("id2", MappingType.Short) + .put("blob", MappingType.Binary) + .build().entrySet().stream().collect(Collectors.toMap( + e -> e.getKey(), e -> OpenSearchDataType.of(e.getValue()) + ))); + when(client.getIndexMappings("test")).thenReturn(ImmutableMap.of("test", mapping)); // Run more than once to confirm caching logic is covered and can work for (int i = 0; i < 2; i++) { @@ -136,7 +141,7 @@ void getFieldTypes() { allOf( aMapWithSize(13), hasEntry("name", ExprCoreType.STRING), - hasEntry("address", (ExprType) OpenSearchDataType.OPENSEARCH_TEXT), + hasEntry("address", (ExprType) OpenSearchDataType.of(MappingType.Text)), hasEntry("age", ExprCoreType.INTEGER), hasEntry("account_number", ExprCoreType.LONG), hasEntry("balance1", ExprCoreType.FLOAT), @@ -147,11 +152,37 @@ void getFieldTypes() { hasEntry("birthday", ExprCoreType.TIMESTAMP), hasEntry("id1", ExprCoreType.BYTE), hasEntry("id2", ExprCoreType.SHORT), - hasEntry("blob", (ExprType) OpenSearchDataType.OPENSEARCH_BINARY) - )); + hasEntry("blob", (ExprType) OpenSearchDataType.of(MappingType.Binary)) + )); } } + @Test + void checkCacheUsedForFieldMappings() { + when(mapping.getFieldMappings()).thenReturn( + Map.of("name", OpenSearchDataType.of(MappingType.Keyword))); + when(client.getIndexMappings("test")).thenReturn( + ImmutableMap.of("test", mapping)); + + OpenSearchIndex index = new OpenSearchIndex(client, settings, "test"); + assertThat(index.getFieldTypes(), allOf( + aMapWithSize(1), + hasEntry("name", STRING))); + assertThat(index.getFieldOpenSearchTypes(), allOf( + aMapWithSize(1), + hasEntry("name", OpenSearchDataType.of(STRING)))); + + lenient().when(mapping.getFieldMappings()).thenReturn( + Map.of("name", OpenSearchDataType.of(MappingType.Integer))); + + assertThat(index.getFieldTypes(), allOf( + aMapWithSize(1), + hasEntry("name", STRING))); + assertThat(index.getFieldOpenSearchTypes(), allOf( + aMapWithSize(1), + hasEntry("name", OpenSearchDataType.of(STRING)))); + } + @Test void implementRelationOperatorOnly() { when(settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT)).thenReturn(200); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanOptimizationTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanOptimizationTest.java index 363727cbd3..b90ca8836d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanOptimizationTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanOptimizationTest.java @@ -38,6 +38,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import lombok.Builder; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.BeforeEach; @@ -60,6 +61,7 @@ import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.HighlightExpression; import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; @@ -552,7 +554,9 @@ private Runnable withAggregationPushedDown( return () -> { verify(requestBuilder, times(1)).pushDownAggregation(Pair.of(aggBuilders, responseParser)); - verify(requestBuilder, times(1)).pushTypeMapping(aggregation.resultTypes); + verify(requestBuilder, times(1)).pushTypeMapping(aggregation.resultTypes + .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, + e -> OpenSearchDataType.of(e.getValue())))); }; } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java index b62d545206..e771e01bce 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java @@ -22,7 +22,6 @@ import static org.opensearch.sql.expression.DSL.named; import static org.opensearch.sql.expression.DSL.ref; import static org.opensearch.sql.expression.DSL.span; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; import static org.opensearch.sql.opensearch.utils.Utils.agg; import static org.opensearch.sql.opensearch.utils.Utils.avg; import static org.opensearch.sql.opensearch.utils.Utils.group; @@ -52,6 +51,8 @@ import org.opensearch.sql.expression.aggregation.AvgAggregator; import org.opensearch.sql.expression.aggregation.CountAggregator; import org.opensearch.sql.expression.aggregation.NamedAggregator; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -142,8 +143,8 @@ void should_build_type_mapping_for_field_reference() { named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER))), Arrays.asList(named("name", ref("name", STRING)))), containsInAnyOrder( - map("avg(age)", INTEGER), - map("name", STRING) + map("avg(age)", OpenSearchDataType.of(INTEGER)), + map("name", OpenSearchDataType.of(STRING)) )); } @@ -177,7 +178,8 @@ void should_build_composite_aggregation_for_field_reference_of_keyword() { buildQuery( Arrays.asList( named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER))), - Arrays.asList(named("name", ref("name", OPENSEARCH_TEXT_KEYWORD))))); + Arrays.asList(named("name", ref("name", OpenSearchTextType.of(Map.of("words", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword)))))))); } @Test @@ -185,10 +187,10 @@ void should_build_type_mapping_for_field_reference_of_keyword() { assertThat( buildTypeMapping(Arrays.asList( named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER))), - Arrays.asList(named("name", ref("name", OPENSEARCH_TEXT_KEYWORD)))), + Arrays.asList(named("name", ref("name", STRING)))), containsInAnyOrder( - map("avg(age)", INTEGER), - map("name", OPENSEARCH_TEXT_KEYWORD) + map("avg(age)", OpenSearchDataType.of(INTEGER)), + map("name", OpenSearchDataType.of(STRING)) )); } @@ -288,8 +290,8 @@ void should_build_type_mapping_for_expression() { Arrays.asList(DSL.abs(ref("balance", INTEGER))), INTEGER))), Arrays.asList(named("age", DSL.asin(ref("age", INTEGER))))), containsInAnyOrder( - map("avg(balance)", INTEGER), - map("age", DOUBLE) + map("avg(balance)", OpenSearchDataType.of(INTEGER)), + map("age", OpenSearchDataType.of(DOUBLE)) )); } @@ -388,7 +390,7 @@ void should_build_filter_aggregation_group_by() { Arrays.asList(named("avg(age) filter(where age > 34)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER) .condition(DSL.greater(ref("age", INTEGER), literal(20))))), - Arrays.asList(named(ref("gender", STRING))))); + Arrays.asList(named(ref("gender", OpenSearchDataType.of(STRING)))))); } @Test @@ -399,7 +401,7 @@ void should_build_type_mapping_without_bucket() { Arrays.asList(ref("balance", INTEGER)), INTEGER))), Collections.emptyList()), containsInAnyOrder( - map("avg(balance)", INTEGER) + map("avg(balance)", OpenSearchDataType.of(INTEGER)) )); } @@ -609,7 +611,7 @@ private String buildQuery( .toPrettyString(); } - private Set> buildTypeMapping( + private Set> buildTypeMapping( List namedAggregatorList, List groupByList) { return queryBuilder.buildTypeMapping(namedAggregatorList, groupByList).entrySet(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/ExpressionAggregationScriptTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/ExpressionAggregationScriptTest.java index d8abb73140..b98bc538ab 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/ExpressionAggregationScriptTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/ExpressionAggregationScriptTest.java @@ -17,8 +17,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.literal; import static org.opensearch.sql.expression.DSL.ref; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; import com.google.common.collect.ImmutableMap; import java.time.LocalDate; @@ -38,10 +36,11 @@ import org.opensearch.search.lookup.SearchLookup; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDatetimeValue; -import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @ExtendWith(MockitoExtension.class) @@ -79,7 +78,9 @@ void can_execute_expression_with_text_keyword_field() { assertThat() .docValues("name.keyword", "John") .evaluate( - DSL.equal(ref("name", OPENSEARCH_TEXT_KEYWORD), literal("John"))) + DSL.equal(ref("name", OpenSearchTextType.of(Map.of("words", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword)))), + literal("John"))) .shouldMatch(true); } @@ -151,7 +152,7 @@ void can_execute_expression_interpret_timestamps_for_aggregation() { void can_execute_expression_interpret_non_core_type_for_aggregation() { assertThat() .docValues("text", "pewpew") - .evaluate(ref("text", OPENSEARCH_TEXT)) + .evaluate(ref("text", OpenSearchTextType.of())) .shouldMatch("pewpew"); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java index 32ab263b8f..56854010a6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java @@ -18,10 +18,10 @@ import static org.opensearch.sql.expression.DSL.literal; import static org.opensearch.sql.expression.DSL.named; import static org.opensearch.sql.expression.DSL.ref; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; import java.util.Arrays; import java.util.List; +import java.util.Map; import lombok.SneakyThrows; import org.apache.commons.lang3.tuple.Triple; import org.junit.jupiter.api.BeforeEach; @@ -46,6 +46,8 @@ import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.parse.ParseExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -112,7 +114,8 @@ void should_build_bucket_with_keyword_field() { + "}", buildQuery( Arrays.asList( - asc(named("name", ref("name", OPENSEARCH_TEXT_KEYWORD)))))); + asc(named("name", ref("name", OpenSearchTextType.of(Map.of("words", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword))))))))); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/ExpressionFilterScriptTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/ExpressionFilterScriptTest.java index ee63c2f0c0..b21ab95927 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/ExpressionFilterScriptTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/ExpressionFilterScriptTest.java @@ -21,7 +21,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.expression.DSL.literal; import static org.opensearch.sql.expression.DSL.ref; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; import com.google.common.collect.ImmutableMap; import java.time.ZonedDateTime; @@ -44,6 +43,8 @@ import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.LiteralExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @ExtendWith(MockitoExtension.class) @@ -88,7 +89,9 @@ void can_execute_expression_with_text_keyword_field() { assertThat() .docValues("name.keyword", "John") .filterBy( - DSL.equal(ref("name", OPENSEARCH_TEXT_KEYWORD), literal("John"))) + DSL.equal(ref("name", OpenSearchTextType.of(Map.of("words", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword)))), + literal("John"))) .shouldMatch(); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index 2ad1f59d39..96245909a4 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -25,7 +25,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.expression.DSL.literal; import static org.opensearch.sql.expression.DSL.ref; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; import com.google.common.collect.ImmutableMap; import java.util.LinkedHashMap; @@ -54,6 +53,8 @@ import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -274,7 +275,9 @@ void should_use_keyword_for_multi_field_in_equality_expression() { + "}", buildQuery( DSL.equal( - ref("name", OPENSEARCH_TEXT_KEYWORD), literal("John")))); + ref("name", OpenSearchTextType.of(Map.of("words", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword)))), + literal("John")))); } @Test @@ -291,7 +294,9 @@ void should_use_keyword_for_multi_field_in_like_expression() { + "}", buildQuery( DSL.like( - ref("name", OPENSEARCH_TEXT_KEYWORD), literal("John%")))); + ref("name", OpenSearchTextType.of(Map.of("words", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword)))), + literal("John%")))); } @Test @@ -315,7 +320,7 @@ void should_build_match_query_with_default_parameters() { buildQuery( DSL.match( DSL.namedArgument("field", - new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("message", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query"))))); } @@ -344,7 +349,7 @@ void should_build_match_query_with_custom_parameters() { buildQuery( DSL.match( DSL.namedArgument("field", - new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("message", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query")), DSL.namedArgument("operator", literal("AND")), DSL.namedArgument("analyzer", literal("keyword")), @@ -364,7 +369,7 @@ void should_build_match_query_with_custom_parameters() { void match_invalid_parameter() { FunctionExpression expr = DSL.match( DSL.namedArgument("field", - new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("message", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query")), DSL.namedArgument("invalid_parameter", literal("invalid_value"))); var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); @@ -438,7 +443,7 @@ void should_build_match_phrase_query_with_default_parameters() { buildQuery( DSL.match_phrase( DSL.namedArgument("field", - new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("message", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query"))))); } @@ -631,7 +636,7 @@ void should_build_match_phrase_query_with_custom_parameters() { buildQuery( DSL.match_phrase( DSL.namedArgument("field", - new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("message", OpenSearchTextType.of())), DSL.namedArgument("boost", literal("1.2")), DSL.namedArgument("query", literal("search query")), DSL.namedArgument("analyzer", literal("keyword")), @@ -643,7 +648,7 @@ void should_build_match_phrase_query_with_custom_parameters() { void wildcard_query_invalid_parameter() { FunctionExpression expr = DSL.wildcard_query( DSL.namedArgument("field", - new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query*")), DSL.namedArgument("invalid_parameter", literal("invalid_value"))); assertThrows(SemanticCheckException.class, () -> buildQuery(expr), @@ -663,7 +668,7 @@ void wildcard_query_convert_sql_wildcard_to_lucene() { + "}", buildQuery(DSL.wildcard_query( DSL.namedArgument("field", - new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query%"))))); assertJsonEquals("{\n" @@ -676,7 +681,7 @@ void wildcard_query_convert_sql_wildcard_to_lucene() { + "}", buildQuery(DSL.wildcard_query( DSL.namedArgument("field", - new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query_"))))); } @@ -692,7 +697,7 @@ void wildcard_query_escape_wildcards_characters() { + "}", buildQuery(DSL.wildcard_query( DSL.namedArgument("field", - new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query\\%"))))); assertJsonEquals("{\n" @@ -705,7 +710,7 @@ void wildcard_query_escape_wildcards_characters() { + "}", buildQuery(DSL.wildcard_query( DSL.namedArgument("field", - new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query\\_"))))); assertJsonEquals("{\n" @@ -718,7 +723,7 @@ void wildcard_query_escape_wildcards_characters() { + "}", buildQuery(DSL.wildcard_query( DSL.namedArgument("field", - new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query\\*"))))); assertJsonEquals("{\n" @@ -731,7 +736,7 @@ void wildcard_query_escape_wildcards_characters() { + "}", buildQuery(DSL.wildcard_query( DSL.namedArgument("field", - new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query\\?"))))); } @@ -747,7 +752,7 @@ void should_build_wildcard_query_with_default_parameters() { + "}", buildQuery(DSL.wildcard_query( DSL.namedArgument("field", - new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query*"))))); } @@ -765,7 +770,7 @@ void should_build_wildcard_query_query_with_custom_parameters() { + "}", buildQuery(DSL.wildcard_query( DSL.namedArgument("field", - new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query*")), DSL.namedArgument("boost", literal("0.6")), DSL.namedArgument("case_insensitive", literal("true")), @@ -1104,7 +1109,7 @@ void simple_query_string_invalid_parameter() { void match_phrase_invalid_parameter() { FunctionExpression expr = DSL.match_phrase( DSL.namedArgument("field", - new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("message", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query")), DSL.namedArgument("invalid_parameter", literal("invalid_value"))); var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); @@ -1114,7 +1119,7 @@ void match_phrase_invalid_parameter() { @Test void relevancy_func_invalid_arg_values() { final var field = DSL.namedArgument("field", - new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)); + new ReferenceExpression("message", OpenSearchTextType.of())); final var fields = DSL.namedArgument("fields", DSL.literal( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( "field1", ExprValueUtils.floatValue(1.F), @@ -1193,7 +1198,7 @@ void should_build_match_bool_prefix_query_with_default_parameters() { buildQuery( DSL.match_bool_prefix( DSL.namedArgument("field", - new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("message", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query"))))); } @@ -1238,7 +1243,7 @@ void should_build_match_phrase_prefix_query_with_default_parameters() { buildQuery( DSL.match_phrase_prefix( DSL.namedArgument("field", - new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("message", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query"))))); } @@ -1260,7 +1265,7 @@ void should_build_match_phrase_prefix_query_with_non_default_parameters() { buildQuery( DSL.match_phrase_prefix( DSL.namedArgument("field", - new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("message", OpenSearchTextType.of())), DSL.namedArgument("query", literal("search query")), DSL.namedArgument("boost", literal("1.2")), DSL.namedArgument("max_expansions", literal("42")), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchBoolPrefixQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchBoolPrefixQueryTest.java index 34ef29f091..6906619065 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchBoolPrefixQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchBoolPrefixQueryTest.java @@ -26,7 +26,7 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; -import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchBoolPrefixQuery; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -36,7 +36,7 @@ public class MatchBoolPrefixQueryTest { static Stream> generateValidData() { NamedArgumentExpression field = DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)); + new ReferenceExpression("field_value", OpenSearchTextType.of())); NamedArgumentExpression query = DSL.namedArgument("query", DSL.literal("query_value")); return List.of( DSL.namedArgument("fuzziness", DSL.literal("AUTO")), @@ -62,7 +62,7 @@ public void test_valid_arguments(List validArgs) { public void test_valid_when_two_arguments() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", "query_value")); Assertions.assertNotNull(matchBoolPrefixQuery.build(new MatchExpression(arguments))); } @@ -85,7 +85,7 @@ public void test_SyntaxCheckException_when_one_argument() { public void test_SemanticCheckException_when_invalid_argument() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", "query_value"), DSL.namedArgument("unsupported", "unsupported_value")); Assertions.assertThrows(SemanticCheckException.class, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhrasePrefixQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhrasePrefixQueryTest.java index e02f112677..0defee0008 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhrasePrefixQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhrasePrefixQueryTest.java @@ -23,7 +23,7 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; -import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchPhrasePrefixQuery; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -42,7 +42,7 @@ public void test_SyntaxCheckException_when_no_arguments() { @Test public void test_SyntaxCheckException_when_one_argument() { List arguments = List.of(DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD))); + new ReferenceExpression("test", OpenSearchTextType.of()))); assertThrows(SyntaxCheckException.class, () -> matchPhrasePrefixQuery.build(new MatchPhraseExpression(arguments))); } @@ -51,7 +51,7 @@ public void test_SyntaxCheckException_when_one_argument() { public void test_SyntaxCheckException_when_invalid_parameter() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("test", OpenSearchTextType.of())), DSL.namedArgument("query", "test2"), DSL.namedArgument("unsupported", "3")); Assertions.assertThrows(SemanticCheckException.class, @@ -62,7 +62,7 @@ public void test_SyntaxCheckException_when_invalid_parameter() { public void test_analyzer_parameter() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("analyzer", "standard") ); @@ -73,7 +73,7 @@ public void test_analyzer_parameter() { public void build_succeeds_with_two_arguments() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("test", OpenSearchTextType.of())), DSL.namedArgument("query", "test2")); Assertions.assertNotNull(matchPhrasePrefixQuery.build(new MatchPhraseExpression(arguments))); } @@ -82,7 +82,7 @@ public void build_succeeds_with_two_arguments() { public void test_slop_parameter() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("slop", "2") ); @@ -93,7 +93,7 @@ public void test_slop_parameter() { public void test_zero_terms_query_parameter() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "ALL") ); @@ -104,7 +104,7 @@ public void test_zero_terms_query_parameter() { public void test_zero_terms_query_parameter_lower_case() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "all") ); @@ -115,7 +115,7 @@ public void test_zero_terms_query_parameter_lower_case() { public void test_boost_parameter() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("test", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("boost", "0.1") ); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java index dd6296279c..20ecb869ba 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java @@ -23,7 +23,7 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; -import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchPhraseQuery; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -44,7 +44,7 @@ public void test_SyntaxCheckException_when_no_arguments() { @Test public void test_SyntaxCheckException_when_one_argument() { List arguments = List.of(DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD))); + new ReferenceExpression("test", OpenSearchTextType.of()))); assertThrows(SyntaxCheckException.class, () -> matchPhraseQuery.build(new MatchPhraseExpression(arguments))); } @@ -53,7 +53,7 @@ public void test_SyntaxCheckException_when_one_argument() { public void test_SyntaxCheckException_when_invalid_parameter() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("test", OpenSearchTextType.of())), DSL.namedArgument("query", "test2"), DSL.namedArgument("unsupported", "3")); Assertions.assertThrows(SemanticCheckException.class, @@ -64,7 +64,7 @@ public void test_SyntaxCheckException_when_invalid_parameter() { public void test_analyzer_parameter() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("analyzer", "standard") ); @@ -75,7 +75,7 @@ public void test_analyzer_parameter() { public void build_succeeds_with_two_arguments() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("test", OpenSearchTextType.of())), DSL.namedArgument("query", "test2")); Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression(arguments))); } @@ -84,7 +84,7 @@ public void build_succeeds_with_two_arguments() { public void test_slop_parameter() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("slop", "2") ); @@ -95,7 +95,7 @@ public void test_slop_parameter() { public void test_zero_terms_query_parameter() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "ALL") ); @@ -106,7 +106,7 @@ public void test_zero_terms_query_parameter() { public void test_zero_terms_query_parameter_lower_case() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "all") ); @@ -124,7 +124,7 @@ public void test_SyntaxCheckException_when_no_arguments_match_phrase_syntax() { @Test public void test_SyntaxCheckException_when_one_argument_match_phrase_syntax() { List arguments = List.of(DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD))); + new ReferenceExpression("test", OpenSearchTextType.of()))); assertThrows(SyntaxCheckException.class, () -> matchPhraseQuery.build(new MatchPhraseExpression( arguments, matchPhraseWithUnderscoreName))); @@ -135,7 +135,7 @@ public void test_SyntaxCheckException_when_one_argument_match_phrase_syntax() { public void test_SyntaxCheckException_when_invalid_parameter_match_phrase_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("test", OpenSearchTextType.of())), DSL.namedArgument("query", "test2"), DSL.namedArgument("unsupported", "3")); Assertions.assertThrows(SemanticCheckException.class, @@ -147,7 +147,7 @@ public void test_SyntaxCheckException_when_invalid_parameter_match_phrase_syntax public void test_analyzer_parameter_match_phrase_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("analyzer", "standard") ); @@ -159,7 +159,7 @@ public void test_analyzer_parameter_match_phrase_syntax() { public void build_succeeds_with_two_arguments_match_phrase_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("test", OpenSearchTextType.of())), DSL.namedArgument("query", "test2")); Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( arguments, matchPhraseWithUnderscoreName))); @@ -169,7 +169,7 @@ public void build_succeeds_with_two_arguments_match_phrase_syntax() { public void test_slop_parameter_match_phrase_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("slop", "2") ); @@ -181,7 +181,7 @@ public void test_slop_parameter_match_phrase_syntax() { public void test_zero_terms_query_parameter_match_phrase_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "ALL") ); @@ -193,7 +193,7 @@ public void test_zero_terms_query_parameter_match_phrase_syntax() { public void test_zero_terms_query_parameter_lower_case_match_phrase_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "all") ); @@ -212,7 +212,7 @@ public void test_SyntaxCheckException_when_no_arguments_matchphrase_syntax() { @Test public void test_SyntaxCheckException_when_one_argument_matchphrase_syntax() { List arguments = List.of(DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD))); + new ReferenceExpression("test", OpenSearchTextType.of()))); assertThrows(SyntaxCheckException.class, () -> matchPhraseQuery.build(new MatchPhraseExpression( arguments, matchPhraseQueryName))); @@ -223,7 +223,7 @@ public void test_SyntaxCheckException_when_one_argument_matchphrase_syntax() { public void test_SyntaxCheckException_when_invalid_parameter_matchphrase_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("test", OpenSearchTextType.of())), DSL.namedArgument("query", "test2"), DSL.namedArgument("unsupported", "3")); Assertions.assertThrows(SemanticCheckException.class, @@ -235,7 +235,7 @@ public void test_SyntaxCheckException_when_invalid_parameter_matchphrase_syntax( public void test_analyzer_parameter_matchphrase_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("analyzer", "standard") ); @@ -247,7 +247,7 @@ public void test_analyzer_parameter_matchphrase_syntax() { public void build_succeeds_with_two_arguments_matchphrase_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("test", OpenSearchTextType.of())), DSL.namedArgument("query", "test2")); Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( arguments, matchPhraseQueryName))); @@ -257,7 +257,7 @@ public void build_succeeds_with_two_arguments_matchphrase_syntax() { public void test_slop_parameter_matchphrase_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("slop", "2") ); @@ -269,7 +269,7 @@ public void test_slop_parameter_matchphrase_syntax() { public void test_zero_terms_query_parameter_matchphrase_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "ALL") ); @@ -281,7 +281,7 @@ public void test_zero_terms_query_parameter_matchphrase_syntax() { public void test_zero_terms_query_parameter_lower_case_matchphrase_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("t1", OpenSearchTextType.of())), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "all") ); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchQueryTest.java index f7d1f1aa9a..ddabb3820e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchQueryTest.java @@ -26,7 +26,7 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; -import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchQuery; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -42,84 +42,84 @@ static Stream> generateValidData() { return Stream.of( List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("analyzer", DSL.literal("standard")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("auto_generate_synonyms_phrase_query", DSL.literal("true")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("fuzziness", DSL.literal("AUTO")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("max_expansions", DSL.literal("50")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("prefix_length", DSL.literal("0")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("fuzzy_transpositions", DSL.literal("true")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("fuzzy_rewrite", DSL.literal("constant_score")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("lenient", DSL.literal("false")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("operator", DSL.literal("OR")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("minimum_should_match", DSL.literal("3")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("zero_terms_query", DSL.literal("NONE")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("zero_terms_query", DSL.literal("none")) ), List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("boost", DSL.literal("1")) ) @@ -150,7 +150,7 @@ public void test_SyntaxCheckException_when_one_argument() { public void test_SemanticCheckException_when_invalid_parameter() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), namedArgument("query", "query_value"), namedArgument("unsupported", "unsupported_value")); Assertions.assertThrows(SemanticCheckException.class, @@ -184,7 +184,7 @@ public void test_SyntaxCheckException_when_one_argument_matchquery_syntax() { public void test_SemanticCheckException_when_invalid_parameter_matchquery_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), namedArgument("query", "query_value"), namedArgument("unsupported", "unsupported_value")); Assertions.assertThrows(SemanticCheckException.class, @@ -219,7 +219,7 @@ public void test_SyntaxCheckException_when_one_argument_match_query_syntax() { public void test_SemanticCheckException_when_invalid_parameter_match_query_syntax() { List arguments = List.of( DSL.namedArgument("field", - new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("field_value", OpenSearchTextType.of())), namedArgument("query", "query_value"), namedArgument("unsupported", "unsupported_value")); Assertions.assertThrows(SemanticCheckException.class, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/WildcardQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/WildcardQueryTest.java index 684036595c..98bd7c5784 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/WildcardQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/WildcardQueryTest.java @@ -25,7 +25,7 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; -import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.WildcardQuery; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -37,7 +37,7 @@ static Stream> generateValidData() { return Stream.of( List.of( namedArgument("field", - new ReferenceExpression("title", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("title", OpenSearchTextType.of())), namedArgument("query", "query_value*"), namedArgument("boost", "0.7"), namedArgument("case_insensitive", "false"), @@ -63,7 +63,7 @@ public void test_SyntaxCheckException_when_no_arguments() { @Test public void test_SyntaxCheckException_when_one_argument() { List arguments = List.of(namedArgument("field", - new ReferenceExpression("title", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD))); + new ReferenceExpression("title", OpenSearchTextType.of()))); assertThrows(SyntaxCheckException.class, () -> wildcardQueryQuery.build(new WildcardQueryExpression(arguments))); } @@ -72,7 +72,7 @@ public void test_SyntaxCheckException_when_one_argument() { public void test_SemanticCheckException_when_invalid_parameter() { List arguments = List.of( namedArgument("field", - new ReferenceExpression("title", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression("title", OpenSearchTextType.of())), namedArgument("query", "query_value*"), namedArgument("unsupported", "unsupported_value")); Assertions.assertThrows(SemanticCheckException.class, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java index 67f22178bc..3628dc8abc 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java @@ -21,6 +21,7 @@ import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; class SingleFieldQueryTest { SingleFieldQuery query; @@ -42,7 +43,9 @@ void createQueryBuilderTestTypeTextKeyword() { String sampleField = "fieldA"; query.createQueryBuilder(List.of(DSL.namedArgument("field", - new ReferenceExpression(sampleField, OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + new ReferenceExpression(sampleField, + OpenSearchTextType.of(Map.of("words", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword))))), DSL.namedArgument("query", new LiteralExpression(ExprValueUtils.stringValue(sampleQuery))))); @@ -56,7 +59,7 @@ void createQueryBuilderTestTypeText() { String sampleField = "fieldA"; query.createQueryBuilder(List.of(DSL.namedArgument("field", - new ReferenceExpression(sampleField, OpenSearchDataType.OPENSEARCH_TEXT)), + new ReferenceExpression(sampleField, OpenSearchTextType.of())), DSL.namedArgument("query", new LiteralExpression(ExprValueUtils.stringValue(sampleQuery))))); diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/JdbcResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/JdbcResponseFormatterTest.java index 96b383ac0c..a6671c66f8 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/JdbcResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/JdbcResponseFormatterTest.java @@ -17,14 +17,13 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.executor.ExecutionEngine.Schema; import static org.opensearch.sql.executor.ExecutionEngine.Schema.Column; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT; -import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.COMPACT; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonParser; import java.util.Arrays; +import java.util.Map; import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; @@ -32,6 +31,8 @@ import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.protocol.response.QueryResult; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -44,8 +45,9 @@ void format_response() { QueryResult response = new QueryResult( new Schema(ImmutableList.of( new Column("name", "name", STRING), - new Column("address1", "address1", OPENSEARCH_TEXT), - new Column("address2", "address2", OPENSEARCH_TEXT_KEYWORD), + new Column("address1", "address1", OpenSearchTextType.of()), + new Column("address2", "address2", OpenSearchTextType.of(Map.of("words", + OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword)))), new Column("location", "location", STRUCT), new Column("employer", "employer", ARRAY), new Column("age", "age", INTEGER))),