From 9c8ba3d6bc564693838fec44ae50ff07f46ccfc5 Mon Sep 17 00:00:00 2001 From: Lantao Jin Date: Fri, 28 Jun 2024 18:18:41 +0800 Subject: [PATCH] Allow metadata fields in PPL query Signed-off-by: Lantao Jin --- .../sql/analysis/ExpressionAnalyzer.java | 3 +-- docs/user/ppl/general/identifiers.rst | 25 ++++++++++++++++++ .../opensearch/sql/ppl/FieldsCommandIT.java | 15 +++++++++++ .../opensearch/sql/ppl/WhereCommandIT.java | 8 ++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 5 +++- .../ppl/parser/AstExpressionBuilderTest.java | 26 +++++++++++++++++++ 6 files changed, 79 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index 5a8d6fe976..eab0eff03c 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -356,8 +356,7 @@ public Expression visitWhen(When node, AnalysisContext context) { @Override public Expression visitField(Field node, AnalysisContext context) { - String attr = node.getField().toString(); - return visitIdentifier(attr, context); + return visitQualifiedName((QualifiedName) node.getField(), context); } @Override diff --git a/docs/user/ppl/general/identifiers.rst b/docs/user/ppl/general/identifiers.rst index 51fc36c40f..30cd2c2235 100644 --- a/docs/user/ppl/general/identifiers.rst +++ b/docs/user/ppl/general/identifiers.rst @@ -161,3 +161,28 @@ Query delimited multiple indices seperated by ``,``:: | 5 | +-----------+ +Metadata Identifiers +==================== + +Description +----------- + +One can also provide meta-field name(s) to retrieve reserved-fields (beginning with underscore) from OpenSearch documents. Meta-fields are not output +as default field list (`search source=`) and must be explicitly included to be returned. + +Examples +--------- + +Query metadata fields:: + + os> source=accounts | fields firstname, lastname, _index, _sort; + fetched rows / total rows = 4/4 + +-------------+------------+----------+---------+ + | firstname | lastname | _index | _sort | + |-------------+------------+----------+---------| + | Amber | Duke | accounts | -2 | + | Hattie | Bond | accounts | -2 | + | Nanette | Bates | accounts | -2 | + | Dale | Adams | accounts | -2 | + +-------------+------------+----------+---------+ + diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java index e8a287c80e..effebe28fa 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java @@ -68,4 +68,19 @@ public void testSelectDateTypeField() throws IOException { rows("2018-08-19 00:00:00"), rows("2018-08-11 00:00:00")); } + + @Test + public void testMetadataFields() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | fields firstname, _index", TEST_INDEX_ACCOUNT)); + verifyColumn(result, columnName("firstname"), columnName("_index")); + } + + @Test + public void testDelimitedMetadataFields() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s | fields firstname, `_id`, `_index`", TEST_INDEX_ACCOUNT)); + verifyColumn(result, columnName("firstname"), columnName("_id"), columnName("_index")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java index d56f9ffe32..4dd6d9e35e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java @@ -98,4 +98,12 @@ public void testIsNotNullFunction() throws IOException { TEST_INDEX_BANK_WITH_NULL_VALUES)); verifyDataRows(result, rows("Amber JOHnny")); } + + @Test + public void testWhereWithMetadataFields() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s | where _id='1' | fields firstname", TEST_INDEX_ACCOUNT)); + verifyDataRows(result, rows("Amber")); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 9f707c13cd..a3e7b5ff35 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -389,7 +389,6 @@ INTEGER_LITERAL: DEC_DIGIT+; DECIMAL_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+; fragment DATE_SUFFIX: ([\-.][*0-9]+)+; -fragment ID_LITERAL: [@*A-Z]+?[*A-Z_\-0-9]*; fragment CLUSTER_PREFIX_LITERAL: [*A-Z]+?[*A-Z_\-0-9]* COLON; ID_DATE_SUFFIX: CLUSTER_PREFIX_LITERAL? ID_LITERAL DATE_SUFFIX; DQUOTA_STRING: '"' ( '\\'. | '""' | ~('"'| '\\') )* '"'; @@ -397,5 +396,9 @@ SQUOTA_STRING: '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\'' BQUOTA_STRING: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`'; fragment DEC_DIGIT: [0-9]; +// Identifiers cannot start with a single '_' since this an OpenSearch reserved +// metadata field. Two underscores (or more) is acceptable, such as '__field'. +fragment ID_LITERAL: ([@*A-Z_])+?[*A-Z_\-0-9]*; + ERROR_RECOGNITION: . -> channel(ERRORCHANNEL); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index 7bcb87d193..4dbc82ee9e 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -532,6 +532,32 @@ public void canBuildKeywordsAsIdentInQualifiedName() { projectWithArg(relation("test"), defaultFieldsArgs(), field("timestamp"))); } + @Test + public void canBuildMetaDataFieldAsQualifiedName() { + assertEqual( + "source=test | fields _id, _index, _sort, _maxscore", + projectWithArg( + relation("test"), + defaultFieldsArgs(), + field("_id"), + field("_index"), + field("_sort"), + field("_maxscore"))); + } + + @Test + public void canBuildNonMetaDataFieldAsQualifiedName() { + assertEqual( + "source=test | fields id, __id, _routing, ___field", + projectWithArg( + relation("test"), + defaultFieldsArgs(), + field("id"), + field("__id"), + field("_routing"), + field("___field"))); + } + @Test public void canBuildMatchRelevanceFunctionWithArguments() { assertEqual(