Add typeof function. (#867)

* Add `typeof` function. Signed-off-by: Yury-Fridlyand <yuryf@bitquilltech.com> * Address PR feedback. * Move function definition in ANTLR grammar into a separate group; * Move SQL integration tests outside of legacy block; * Extend integration tests. Signed-off-by: Yury-Fridlyand <yuryf@bitquilltech.com> * Fix indentation. Signed-off-by: Yury-Fridlyand <yuryf@bitquilltech.com> * Implement `typeof` as a bespoke `FunctionResolver` and `FunctionExpression`. It is able to recognize `OpenSearchDataType` as well. Co-authored-by: MaxKsyunz <maxk@bitquilltech.com> Signed-off-by: Yury-Fridlyand <yuryf@bitquilltech.com> * Rename `TypeOf` to be a part of `SystemFunctions`. Signed-off-by: Yury-Fridlyand <yuryf@bitquilltech.com> * Add docs. Signed-off-by: Yury-Fridlyand <yuryf@bitquilltech.com> Signed-off-by: Yury-Fridlyand <yuryf@bitquilltech.com> Co-authored-by: MaxKsyunz <maxk@bitquilltech.com>
opensearch-project · Oct 20, 2022 · 1f2e881 · 1f2e881
1 parent 9debb01
commit 1f2e881
Show file tree

Hide file tree

Showing 18 changed files with 422 additions and 5 deletions.
diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java
@@ -13,11 +13,13 @@
 import org.opensearch.sql.data.model.ExprShortValue;
 import org.opensearch.sql.data.model.ExprValue;
 import org.opensearch.sql.data.model.ExprValueUtils;
+import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.data.type.ExprType;
 import org.opensearch.sql.expression.aggregation.Aggregator;
 import org.opensearch.sql.expression.aggregation.NamedAggregator;
 import org.opensearch.sql.expression.conditional.cases.CaseClause;
 import org.opensearch.sql.expression.conditional.cases.WhenClause;
+import org.opensearch.sql.expression.env.Environment;
 import org.opensearch.sql.expression.function.BuiltinFunctionName;
 import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
 import org.opensearch.sql.expression.parse.GrokExpression;
@@ -676,6 +678,11 @@ public FunctionExpression castDatetime(Expression value) {
         .compile(BuiltinFunctionName.CAST_TO_DATETIME.getName(), Arrays.asList(value));
   }
 
+  public FunctionExpression typeof(Expression value) {
+    return (FunctionExpression) repository
+        .compile(BuiltinFunctionName.TYPEOF.getName(), Arrays.asList(value));
+  }
+
   public FunctionExpression match(Expression... args) {
     return compile(BuiltinFunctionName.MATCH, args);
   }

diff --git a/core/src/main/java/org/opensearch/sql/expression/config/ExpressionConfig.java b/core/src/main/java/org/opensearch/sql/expression/config/ExpressionConfig.java
@@ -18,6 +18,7 @@
 import org.opensearch.sql.expression.operator.convert.TypeCastOperator;
 import org.opensearch.sql.expression.operator.predicate.BinaryPredicateOperator;
 import org.opensearch.sql.expression.operator.predicate.UnaryPredicateOperator;
+import org.opensearch.sql.expression.system.SystemFunctions;
 import org.opensearch.sql.expression.text.TextFunction;
 import org.opensearch.sql.expression.window.WindowFunctions;
 import org.springframework.context.annotation.Bean;
@@ -45,6 +46,7 @@ public BuiltinFunctionRepository functionRepository() {
     WindowFunctions.register(builtinFunctionRepository);
     TextFunction.register(builtinFunctionRepository);
     TypeCastOperator.register(builtinFunctionRepository);
+    SystemFunctions.register(builtinFunctionRepository);
     OpenSearchFunctions.register(builtinFunctionRepository);
     return builtinFunctionRepository;
   }

diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java
@@ -197,6 +197,7 @@ public enum BuiltinFunctionName {
   CAST_TO_TIME(FunctionName.of("cast_to_time")),
   CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")),
   CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")),
+  TYPEOF(FunctionName.of("typeof")),
 
   /**
    * Relevance Function.

diff --git a/core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java b/core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.expression.system;
+
+import static org.opensearch.sql.data.type.ExprCoreType.STRING;
+
+import lombok.experimental.UtilityClass;
+import org.apache.commons.lang3.tuple.Pair;
+import org.opensearch.sql.data.model.ExprStringValue;
+import org.opensearch.sql.data.model.ExprValue;
+import org.opensearch.sql.data.type.ExprType;
+import org.opensearch.sql.expression.Expression;
+import org.opensearch.sql.expression.FunctionExpression;
+import org.opensearch.sql.expression.env.Environment;
+import org.opensearch.sql.expression.function.BuiltinFunctionName;
+import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
+import org.opensearch.sql.expression.function.FunctionBuilder;
+import org.opensearch.sql.expression.function.FunctionName;
+import org.opensearch.sql.expression.function.FunctionResolver;
+import org.opensearch.sql.expression.function.FunctionSignature;
+
+@UtilityClass
+public class SystemFunctions {
+  /**
+   * Register TypeOf Operator.
+   */
+  public static void register(BuiltinFunctionRepository repository) {
+    repository.register(typeof());
+  }
+
+  // Auxiliary function useful for debugging
+  private static FunctionResolver typeof() {
+    return new FunctionResolver() {
+      @Override
+      public Pair<FunctionSignature, FunctionBuilder> resolve(
+          FunctionSignature unresolvedSignature) {
+        return Pair.of(unresolvedSignature,
+            arguments -> new FunctionExpression(BuiltinFunctionName.TYPEOF.getName(), arguments) {
+              @Override
+              public ExprValue valueOf(Environment<Expression, ExprValue> valueEnv) {
+                return new ExprStringValue(getArguments().get(0).type().toString());
+              }
+
+              @Override
+              public ExprType type() {
+                return STRING;
+              }
+            });
+      }
+
+      @Override
+      public FunctionName getFunctionName() {
+        return BuiltinFunctionName.TYPEOF.getName();
+      }
+    };
+  }
+}
diff --git a/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.expression.system;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.opensearch.sql.data.type.ExprCoreType.STRING;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.LinkedHashMap;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+import org.opensearch.sql.data.model.AbstractExprValue;
+import org.opensearch.sql.data.model.ExprBooleanValue;
+import org.opensearch.sql.data.model.ExprByteValue;
+import org.opensearch.sql.data.model.ExprCollectionValue;
+import org.opensearch.sql.data.model.ExprDateValue;
+import org.opensearch.sql.data.model.ExprDatetimeValue;
+import org.opensearch.sql.data.model.ExprDoubleValue;
+import org.opensearch.sql.data.model.ExprFloatValue;
+import org.opensearch.sql.data.model.ExprIntegerValue;
+import org.opensearch.sql.data.model.ExprIntervalValue;
+import org.opensearch.sql.data.model.ExprLongValue;
+import org.opensearch.sql.data.model.ExprMissingValue;
+import org.opensearch.sql.data.model.ExprNullValue;
+import org.opensearch.sql.data.model.ExprShortValue;
+import org.opensearch.sql.data.model.ExprStringValue;
+import org.opensearch.sql.data.model.ExprTimeValue;
+import org.opensearch.sql.data.model.ExprTimestampValue;
+import org.opensearch.sql.data.model.ExprTupleValue;
+import org.opensearch.sql.data.model.ExprValue;
+import org.opensearch.sql.data.type.ExprCoreType;
+import org.opensearch.sql.data.type.ExprType;
+import org.opensearch.sql.expression.DSL;
+import org.opensearch.sql.expression.config.ExpressionConfig;
+
+public class SystemFunctionsTest {
+  private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository());
+
+  @Test
+  void typeof() {
+    assertEquals(STRING, dsl.typeof(DSL.literal(1)).type());
+
+    assertEquals("ARRAY", typeofGetValue(new ExprCollectionValue(List.of())));
+    assertEquals("BOOLEAN", typeofGetValue(ExprBooleanValue.of(false)));
+    assertEquals("BYTE", typeofGetValue(new ExprByteValue(0)));
+    assertEquals("DATE", typeofGetValue(new ExprDateValue(LocalDate.now())));
+    assertEquals("DATETIME", typeofGetValue(new ExprDatetimeValue(LocalDateTime.now())));
+    assertEquals("DOUBLE", typeofGetValue(new ExprDoubleValue(0)));
+    assertEquals("FLOAT", typeofGetValue(new ExprFloatValue(0)));
+    assertEquals("INTEGER", typeofGetValue(new ExprIntegerValue(0)));
+    assertEquals("INTERVAL", typeofGetValue(new ExprIntervalValue(Duration.ofDays(0))));
+    assertEquals("LONG", typeofGetValue(new ExprLongValue(0)));
+    assertEquals("SHORT", typeofGetValue(new ExprShortValue(0)));
+    assertEquals("STRING", typeofGetValue(new ExprStringValue("")));
+    assertEquals("STRUCT", typeofGetValue(new ExprTupleValue(new LinkedHashMap<>())));
+    assertEquals("TIME", typeofGetValue(new ExprTimeValue(LocalTime.now())));
+    assertEquals("TIMESTAMP", typeofGetValue(new ExprTimestampValue(Instant.now())));
+    assertEquals("UNDEFINED", typeofGetValue(ExprNullValue.of()));
+    assertEquals("UNDEFINED", typeofGetValue(ExprMissingValue.of()));
+    assertEquals("UNKNOWN", typeofGetValue(new AbstractExprValue() {
+      @Override
+      public int compare(ExprValue other) {
+        return 0;
+      }
+
+      @Override
+      public boolean equal(ExprValue other) {
+        return false;
+      }
+
+      @Override
+      public Object value() {
+        return null;
+      }
+
+      @Override
+      public ExprType type() {
+        return ExprCoreType.UNKNOWN;
+      }
+    }));
+  }
+
+  private String typeofGetValue(ExprValue input) {
+    return dsl.typeof(DSL.literal(input)).valueOf(null).stringValue();
+  }
+}
diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst
@@ -2996,3 +2996,27 @@ Example searching for field Tags::
     | [Winnie-the-<em>Pooh</em>]                   |
     +----------------------------------------------+
 
+System Functions
+================
+
+TYPEOF
+------
+
+Description
+>>>>>>>>>>>
+
+Usage: typeof(expr) function returns name of the data type of the value that is passed to it. This can be helpful for troubleshooting or dynamically constructing SQL queries.
+
+Argument type: ANY
+
+Return type: STRING
+
+Example::
+
+    os> select typeof(DATE('2008-04-14')) as `typeof(date)`, typeof(1) as `typeof(int)`, typeof(now()) as `typeof(now())`, typeof(accounts) as `typeof(column)` from people
+    fetched rows / total rows = 1/1
+    +----------------+---------------+-----------------+------------------+
+    | typeof(date)   | typeof(int)   | typeof(now())   | typeof(column)   |
+    |----------------+---------------+-----------------+------------------|
+    | DATE           | INTEGER       | DATETIME        | STRUCT           |
+    +----------------+---------------+-----------------+------------------+
diff --git a/docs/user/ppl/functions/system.rst b/docs/user/ppl/functions/system.rst
@@ -0,0 +1,31 @@
+================
+System Functions
+================
+
+.. rubric:: Table of contents
+
+.. contents::
+   :local:
+   :depth: 1
+
+TYPEOF
+------
+
+Description
+>>>>>>>>>>>
+
+Usage: typeof(expr) function returns name of the data type of the value that is passed to it. This can be helpful for troubleshooting or dynamically constructing SQL queries.
+
+Argument type: ANY
+
+Return type: STRING
+
+Example::
+
+    os> source=people | eval `typeof(date)` = typeof(DATE('2008-04-14')), `typeof(int)` = typeof(1), `typeof(now())` = typeof(now()), `typeof(column)` = typeof(accounts) | fields `typeof(date)`, `typeof(int)`, `typeof(now())`, `typeof(column)`
+    fetched rows / total rows = 1/1
+    +----------------+---------------+-----------------+------------------+
+    | typeof(date)   | typeof(int)   | typeof(now())   | typeof(column)   |
+    |----------------+---------------+-----------------+------------------|
+    | DATE           | INTEGER       | DATETIME        | STRUCT           |
+    +----------------+---------------+-----------------+------------------+
diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst
@@ -88,6 +88,8 @@ The query start with search command and then flowing a set of command delimited
 
   - `Type Conversion Functions <functions/conversion.rst>`_
 
+  - `System Functions <functions/system.rst>`_
+
 * **Optimization**
 
   - `Optimization <../../user/optimization/optimization.rst>`_

diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java
@@ -52,7 +52,8 @@ public void test_nonnumeric_data_types() throws IOException {
         schema("date_value", "timestamp"),
         schema("ip_value", "ip"),
         schema("object_value", "struct"),
-        schema("nested_value", "array"));
+        schema("nested_value", "array"),
+        schema("geo_point_value", "geo_point"));
   }
 
   @Test
@@ -71,5 +72,4 @@ public void test_long_integer_data_type() throws IOException {
         schema("long1", "long"),
         schema("long2", "long"));
   }
-
 }
diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ppl;
+
+import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NONNUMERIC;
+import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NUMERIC;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC;
+import static org.opensearch.sql.util.MatcherUtils.rows;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
+
+import java.io.IOException;
+import org.json.JSONObject;
+import org.junit.Test;
+
+public class SystemFunctionIT extends PPLIntegTestCase {
+
+  @Override
+  public void init() throws IOException {
+    loadIndex(DATA_TYPE_NUMERIC);
+    loadIndex(DATA_TYPE_NONNUMERIC);
+  }
+
+  @Test
+  public void typeof_sql_types() throws IOException {
+    JSONObject response = executeQuery(String.format("source=%s | eval "
+        + "`str` = typeof('pewpew'), `double` = typeof(1.0),"
+        + "`int` = typeof(12345), `long` = typeof(1234567891011), `interval` = typeof(INTERVAL 2 DAY)"
+        + " | fields `str`, `double`, `int`, `long`, `interval`",
+            TEST_INDEX_DATATYPE_NUMERIC));
+    // TODO: test null in PPL
+    verifyDataRows(response,
+        rows("STRING", "DOUBLE", "INTEGER", "LONG", "INTERVAL"));
+
+    response = executeQuery(String.format("source=%s | eval "
+        + "`timestamp` = typeof(CAST('1961-04-12 09:07:00' AS TIMESTAMP)),"
+        + "`time` = typeof(CAST('09:07:00' AS TIME)),"
+        + "`date` = typeof(CAST('1961-04-12' AS DATE)),"
+        + "`datetime` = typeof(DATETIME('1961-04-12 09:07:00'))"
+        + " | fields `timestamp`, `time`, `date`, `datetime`",
+            TEST_INDEX_DATATYPE_NUMERIC));
+    verifyDataRows(response,
+        rows("TIMESTAMP", "TIME", "DATE", "DATETIME"));
+  }
+
+  @Test
+  public void typeof_opensearch_types() throws IOException {
+    JSONObject response = executeQuery(String.format("source=%s | eval "
+        + "`double` = typeof(double_number), `long` = typeof(long_number),"
+        + "`integer` = typeof(integer_number), `byte` = typeof(byte_number),"
+        + "`short` = typeof(short_number), `float` = typeof(float_number),"
+        + "`half_float` = typeof(half_float_number), `scaled_float` = typeof(scaled_float_number)"
+        + " | fields `double`, `long`, `integer`, `byte`, `short`, `float`, `half_float`, `scaled_float`",
+            TEST_INDEX_DATATYPE_NUMERIC));
+    verifyDataRows(response,
+        rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE"));
+
+    response = executeQuery(String.format("source=%s | eval "
+        + "`text` = typeof(text_value), `date` = typeof(date_value),"
+        + "`boolean` = typeof(boolean_value), `object` = typeof(object_value),"
+        + "`keyword` = typeof(keyword_value), `ip` = typeof(ip_value),"
+        + "`binary` = typeof(binary_value), `geo_point` = typeof(geo_point_value)"
+        // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported
+        //+ ", `nested` = typeof(nested_value)"
+        + " | fields `text`, `date`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`",
+            TEST_INDEX_DATATYPE_NONNUMERIC));
+    verifyDataRows(response,
+        rows("OPENSEARCH_TEXT", "TIMESTAMP", "BOOLEAN", "STRUCT", "STRING",
+                "OPENSEARCH_IP", "OPENSEARCH_BINARY", "OPENSEARCH_GEO_POINT"));
+  }
+}