Allow metadata fields in PPL query

Signed-off-by: Lantao Jin <ltjin@amazon.com>
opensearch-project · Jul 9, 2024 · 9c8ba3d · 9c8ba3d
1 parent f241f34
commit 9c8ba3d
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 3 deletions.
diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java
@@ -356,8 +356,7 @@ public Expression visitWhen(When node, AnalysisContext context) {
 
   @Override
   public Expression visitField(Field node, AnalysisContext context) {
-    String attr = node.getField().toString();
-    return visitIdentifier(attr, context);
+    return visitQualifiedName((QualifiedName) node.getField(), context);
   }
 
   @Override

diff --git a/docs/user/ppl/general/identifiers.rst b/docs/user/ppl/general/identifiers.rst
@@ -161,3 +161,28 @@ Query delimited multiple indices seperated by ``,``::
     | 5         |
     +-----------+
 
+Metadata Identifiers
+====================
+
+Description
+-----------
+
+One can also provide meta-field name(s) to retrieve reserved-fields (beginning with underscore) from OpenSearch documents. Meta-fields are not output
+as default field list (`search source=<index>`) and must be explicitly included to be returned.
+
+Examples
+---------
+
+Query metadata fields::
+
+    os> source=accounts | fields firstname, lastname, _index, _sort;
+    fetched rows / total rows = 4/4
+    +-------------+------------+----------+---------+
+    | firstname   | lastname   | _index   | _sort   |
+    |-------------+------------+----------+---------|
+    | Amber       | Duke       | accounts | -2      |
+    | Hattie      | Bond       | accounts | -2      |
+    | Nanette     | Bates      | accounts | -2      |
+    | Dale        | Adams      | accounts | -2      |
+    +-------------+------------+----------+---------+
+
diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java
@@ -68,4 +68,19 @@ public void testSelectDateTypeField() throws IOException {
         rows("2018-08-19 00:00:00"),
         rows("2018-08-11 00:00:00"));
   }
+
+  @Test
+  public void testMetadataFields() throws IOException {
+    JSONObject result =
+        executeQuery(String.format("source=%s | fields firstname, _index", TEST_INDEX_ACCOUNT));
+    verifyColumn(result, columnName("firstname"), columnName("_index"));
+  }
+
+  @Test
+  public void testDelimitedMetadataFields() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format("source=%s | fields firstname, `_id`, `_index`", TEST_INDEX_ACCOUNT));
+    verifyColumn(result, columnName("firstname"), columnName("_id"), columnName("_index"));
+  }
 }
diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java
@@ -98,4 +98,12 @@ public void testIsNotNullFunction() throws IOException {
                 TEST_INDEX_BANK_WITH_NULL_VALUES));
     verifyDataRows(result, rows("Amber JOHnny"));
   }
+
+  @Test
+  public void testWhereWithMetadataFields() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format("source=%s | where _id='1' | fields firstname", TEST_INDEX_ACCOUNT));
+    verifyDataRows(result, rows("Amber"));
+  }
 }
diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4
@@ -389,13 +389,16 @@ INTEGER_LITERAL:                    DEC_DIGIT+;
 DECIMAL_LITERAL:                    (DEC_DIGIT+)? '.' DEC_DIGIT+;
 
 fragment DATE_SUFFIX:               ([\-.][*0-9]+)+;
-fragment ID_LITERAL:                [@*A-Z]+?[*A-Z_\-0-9]*;
 fragment CLUSTER_PREFIX_LITERAL:    [*A-Z]+?[*A-Z_\-0-9]* COLON;
 ID_DATE_SUFFIX:                     CLUSTER_PREFIX_LITERAL? ID_LITERAL DATE_SUFFIX;
 DQUOTA_STRING:                      '"' ( '\\'. | '""' | ~('"'| '\\') )* '"';
 SQUOTA_STRING:                      '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\'';
 BQUOTA_STRING:                      '`' ( '\\'. | '``' | ~('`'|'\\'))* '`';
 fragment DEC_DIGIT:                 [0-9];
 
+// Identifiers cannot start with a single '_' since this an OpenSearch reserved
+// metadata field.  Two underscores (or more) is acceptable, such as '__field'.
+fragment ID_LITERAL:                ([@*A-Z_])+?[*A-Z_\-0-9]*;
+
 
 ERROR_RECOGNITION:                  .    -> channel(ERRORCHANNEL);
diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java
@@ -532,6 +532,32 @@ public void canBuildKeywordsAsIdentInQualifiedName() {
         projectWithArg(relation("test"), defaultFieldsArgs(), field("timestamp")));
   }
 
+  @Test
+  public void canBuildMetaDataFieldAsQualifiedName() {
+    assertEqual(
+        "source=test | fields _id, _index, _sort, _maxscore",
+        projectWithArg(
+            relation("test"),
+            defaultFieldsArgs(),
+            field("_id"),
+            field("_index"),
+            field("_sort"),
+            field("_maxscore")));
+  }
+
+  @Test
+  public void canBuildNonMetaDataFieldAsQualifiedName() {
+    assertEqual(
+        "source=test | fields id, __id, _routing, ___field",
+        projectWithArg(
+            relation("test"),
+            defaultFieldsArgs(),
+            field("id"),
+            field("__id"),
+            field("_routing"),
+            field("___field")));
+  }
+
   @Test
   public void canBuildMatchRelevanceFunctionWithArguments() {
     assertEqual(