Rework on OpenSearchDataType: parse, store and use mapping informat…

…ion (#1314) (#1455)

* Rework on `OpenSearchDataType`: parse, store and use mapping information (#180)

Rework on `OpenSearchDataType`:
* Add data types for those classes are not defined in `ExprCoreType`.
* Address Bit-Quill#180 (comment)
* Remove `TextKeywordValue`.
* Add changes according to the PR review. Bit-Quill#180
* Update `IndexMapping::parseMapping` function.
* Add `OpenSearchDataType::resolve` function.
* Add new constructor for `OpenSearchTextType`.
* Make `fields` and `properties` in `OpenSearchDataType` readonly. Update tests and mapping parser.
* Move `getFields` from `OpenSearchDataType` to `OpenSearchTextType`. Update tests.
* Rewrite `traverseAndFlatten` according to Bit-Quill#180 (comment)
* Minor comment fix.
* A fix to avoid breaking changes.
* `typeName` and `legacyTypeName` to return different type names.
* Change `typeof` function and corresponding tests.
* Move `convertTextToKeyword` from `ScriptUtils` to `OpenSearchTextType`. Update tests.
* Update UT for `typeof` function.
* Make all instances of `OpenSearchDataType` and of derived types singletones as much as possible.
* Make string representations of all `ExprType`s uppercase.
* Remove functions from `IndexMapping` used in tests only.

Signed-off-by: Yury-Fridlyand <yuryf@bitquilltech.com>
Signed-off-by: Yury-Fridlyand <yury.fridlyand@improving.com>
(cherry picked from commit d44cd39)

Co-authored-by: Yury-Fridlyand <yury.fridlyand@improving.com>

core/src/main/java/org/opensearch/sql/data/model/AbstractExprValue.java

@@ -23,7 +23,7 @@ public int compareTo(ExprValue other) {
     }
     if ((this.isNumber() && other.isNumber())
         || (this.isDateTime() && other.isDateTime())
-        || this.type() == other.type()) {
+        || this.type().equals(other.type())) {
       return compare(other);
     } else {
       throw new ExpressionEvaluationException(

core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java

@@ -80,9 +80,9 @@ public enum ExprCoreType implements ExprType {
    */
   private static final Map<ExprCoreType, String> LEGACY_TYPE_NAME_MAPPING =
       new ImmutableMap.Builder<ExprCoreType, String>()
-          .put(STRUCT, "object")
-          .put(ARRAY, "nested")
-          .put(STRING, "keyword")
+          .put(STRUCT, "OBJECT")
+          .put(ARRAY, "NESTED")
+          .put(STRING, "KEYWORD")
           .build();
 
   private static final Set<ExprType> NUMBER_TYPES =

core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java

@@ -42,7 +42,7 @@ public Pair<FunctionSignature, FunctionBuilder> resolve(
                 new FunctionExpression(BuiltinFunctionName.TYPEOF.getName(), arguments) {
               @Override
               public ExprValue valueOf(Environment<Expression, ExprValue> valueEnv) {
-                return new ExprStringValue(getArguments().get(0).type().toString());
+                return new ExprStringValue(getArguments().get(0).type().legacyTypeName());
               }
 
               @Override

core/src/test/java/org/opensearch/sql/data/type/ExprTypeTest.java

@@ -76,9 +76,9 @@ public void getParent() {
 
   @Test
   void legacyName() {
-    assertEquals("keyword", STRING.legacyTypeName());
-    assertEquals("nested", ARRAY.legacyTypeName());
-    assertEquals("object", STRUCT.legacyTypeName());
+    assertEquals("KEYWORD", STRING.legacyTypeName());
+    assertEquals("NESTED", ARRAY.legacyTypeName());
+    assertEquals("OBJECT", STRUCT.legacyTypeName());
     assertEquals("integer", INTEGER.legacyTypeName().toLowerCase());
   }
 

core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java

@@ -45,7 +45,7 @@ public class SystemFunctionsTest {
   void typeof() {
     assertEquals(STRING, DSL.typeof(DSL.literal(1)).type());
 
-    assertEquals("ARRAY", typeofGetValue(new ExprCollectionValue(List.of())));
+    assertEquals("NESTED", typeofGetValue(new ExprCollectionValue(List.of())));
     assertEquals("BOOLEAN", typeofGetValue(ExprBooleanValue.of(false)));
     assertEquals("BYTE", typeofGetValue(new ExprByteValue(0)));
     assertEquals("DATE", typeofGetValue(new ExprDateValue(LocalDate.now())));
@@ -56,8 +56,8 @@ void typeof() {
     assertEquals("INTERVAL", typeofGetValue(new ExprIntervalValue(Duration.ofDays(0))));
     assertEquals("LONG", typeofGetValue(new ExprLongValue(0)));
     assertEquals("SHORT", typeofGetValue(new ExprShortValue(0)));
-    assertEquals("STRING", typeofGetValue(new ExprStringValue("")));
-    assertEquals("STRUCT", typeofGetValue(new ExprTupleValue(new LinkedHashMap<>())));
+    assertEquals("KEYWORD", typeofGetValue(new ExprStringValue("")));
+    assertEquals("OBJECT", typeofGetValue(new ExprTupleValue(new LinkedHashMap<>())));
     assertEquals("TIME", typeofGetValue(new ExprTimeValue(LocalTime.now())));
     assertEquals("TIMESTAMP", typeofGetValue(new ExprTimestampValue(Instant.now())));
     assertEquals("UNDEFINED", typeofGetValue(ExprNullValue.of()));

docs/user/dql/functions.rst

@@ -4286,7 +4286,7 @@ Example::
     +----------------+---------------+-----------------+------------------+
     | typeof(date)   | typeof(int)   | typeof(now())   | typeof(column)   |
     |----------------+---------------+-----------------+------------------|
-    | DATE           | INTEGER       | DATETIME        | STRUCT           |
+    | DATE           | INTEGER       | DATETIME        | OBJECT           |
     +----------------+---------------+-----------------+------------------+
 
 

docs/user/general/datatypes.rst

@@ -34,7 +34,7 @@ The OpenSearch SQL Engine support the following data types.
 +---------------------+
 | double              |
 +---------------------+
-| string              |
+| keyword             |
 +---------------------+
 | text                |
 +---------------------+
@@ -87,7 +87,7 @@ The table below list the mapping between OpenSearch Data Type, OpenSearch SQL Da
 +-----------------+---------------------+-----------+
 | double          | double              | DOUBLE    |
 +-----------------+---------------------+-----------+
-| keyword         | string              | VARCHAR   |
+| keyword         | keyword             | VARCHAR   |
 +-----------------+---------------------+-----------+
 | text            | text                | VARCHAR   |
 +-----------------+---------------------+-----------+

docs/user/ppl/functions/system.rst

@@ -27,5 +27,5 @@ Example::
     +----------------+---------------+-----------------+------------------+
     | typeof(date)   | typeof(int)   | typeof(now())   | typeof(column)   |
     |----------------+---------------+-----------------+------------------|
-    | DATE           | INTEGER       | DATETIME        | STRUCT           |
+    | DATE           | INTEGER       | DATETIME        | OBJECT           |
     +----------------+---------------+-----------------+------------------+

docs/user/ppl/general/datatypes.rst

@@ -37,8 +37,6 @@ The PPL support the following data types.
 +---------------+
 | string        |
 +---------------+
-| text          |
-+---------------+
 | timestamp     |
 +---------------+
 | datetime      |
@@ -88,7 +86,7 @@ The table below list the mapping between OpenSearch Data Type, PPL Data Type and
 +-----------------+---------------+-----------+
 | keyword         | string        | VARCHAR   |
 +-----------------+---------------+-----------+
-| text            | text          | VARCHAR   |
+| text            | string        | VARCHAR   |
 +-----------------+---------------+-----------+
 | date            | timestamp     | TIMESTAMP |
 +-----------------+---------------+-----------+

integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java

@@ -33,7 +33,7 @@ public void typeof_sql_types() throws IOException {
             TEST_INDEX_DATATYPE_NUMERIC));
     // TODO: test null in PPL
     verifyDataRows(response,
-        rows("STRING", "DOUBLE", "INTEGER", "LONG", "INTERVAL"));
+        rows("KEYWORD", "DOUBLE", "INTEGER", "LONG", "INTERVAL"));
 
     response = executeQuery(String.format("source=%s | eval "
         + "`timestamp` = typeof(CAST('1961-04-12 09:07:00' AS TIMESTAMP)),"
@@ -68,7 +68,7 @@ public void typeof_opensearch_types() throws IOException {
         + " | fields `text`, `date`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`",
             TEST_INDEX_DATATYPE_NONNUMERIC));
     verifyDataRows(response,
-        rows("OPENSEARCH_TEXT", "TIMESTAMP", "BOOLEAN", "STRUCT", "STRING",
-                "OPENSEARCH_IP", "OPENSEARCH_BINARY", "OPENSEARCH_GEO_POINT"));
+        rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD",
+                "IP", "BINARY", "GEO_POINT"));
   }
 }

integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java

@@ -27,7 +27,7 @@ public void typeof_sql_types() {
     JSONObject response = executeJdbcRequest("SELECT typeof('pewpew'), typeof(NULL), typeof(1.0),"
         + "typeof(12345), typeof(1234567891011), typeof(INTERVAL 2 DAY);");
     verifyDataRows(response,
-        rows("STRING", "UNDEFINED", "DOUBLE", "INTEGER", "LONG", "INTERVAL"));
+        rows("KEYWORD", "UNDEFINED", "DOUBLE", "INTEGER", "LONG", "INTERVAL"));
 
     response = executeJdbcRequest("SELECT"
         + " typeof(CAST('1961-04-12 09:07:00' AS TIMESTAMP)),"
@@ -54,7 +54,7 @@ public void typeof_opensearch_types() {
         //+ ", typeof(nested_value)"
         + " from %s;", TEST_INDEX_DATATYPE_NONNUMERIC));
     verifyDataRows(response,
-        rows("OPENSEARCH_TEXT", "TIMESTAMP", "BOOLEAN", "STRUCT", "STRING",
-                "OPENSEARCH_IP", "OPENSEARCH_BINARY", "OPENSEARCH_GEO_POINT"));
+        rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD",
+                "IP", "BINARY", "GEO_POINT"));
   }
 }

opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchBinaryType.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.opensearch.data.type;
+
+import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN;
+
+import lombok.EqualsAndHashCode;
+
+/**
+ * The type of a binary value. See
+ * <a href="https://opensearch.org/docs/latest/opensearch/supported-field-types/binary/">doc</a>
+ */
+@EqualsAndHashCode(callSuper = false)
+public class OpenSearchBinaryType extends OpenSearchDataType {
+
+  private static final OpenSearchBinaryType instance = new OpenSearchBinaryType();
+
+  private OpenSearchBinaryType() {
+    super(MappingType.Binary);
+    exprCoreType = UNKNOWN;
+  }
+
+  public static OpenSearchBinaryType of() {
+    return OpenSearchBinaryType.instance;
+  }
+
+  @Override
+  protected OpenSearchDataType cloneEmpty() {
+    return instance;
+  }
+}