From c49231d7910e1da3901901ecfb0a37544449c1a7 Mon Sep 17 00:00:00 2001 From: Manasvini B S Date: Tue, 18 Jun 2024 12:25:40 -0700 Subject: [PATCH] Add support for custom date format and openSearch date format for date fields as part of Lucene query Github Issue - https://github.com/opensearch-project/sql/issues/2700 Signed-off-by: Manasvini B S --- core/build.gradle | 1 + .../sql/data/model/ExprDateValue.java | 46 ++- .../sql/data/model/ExprTimeValue.java | 49 +++- .../sql/data/model/ExprTimestampValue.java | 64 +++- .../sql/utils/DateTimeFormatters.java | 3 +- .../sql/data/model/DateTimeValueTest.java | 276 +++++++++++++++++- .../datetime/DateTimeFunctionTest.java | 9 +- .../sql/expression/datetime/ExtractTest.java | 5 +- .../sql/expression/datetime/YearweekTest.java | 2 +- .../data/type/OpenSearchDataType.java | 12 +- .../data/type/OpenSearchDateType.java | 4 + .../value/OpenSearchExprValueFactory.java | 7 +- .../opensearch/storage/OpenSearchIndex.java | 4 +- .../script/filter/lucene/LuceneQuery.java | 19 +- .../script/filter/lucene/RangeQuery.java | 11 +- .../script/filter/lucene/TermQuery.java | 10 +- .../data/type/OpenSearchDataTypeTest.java | 12 +- .../data/type/OpenSearchDateTypeTest.java | 42 +-- .../storage/OpenSearchIndexTest.java | 2 +- .../script/filter/FilterQueryBuilderTest.java | 11 +- 20 files changed, 508 insertions(+), 81 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index 655e7d92c2..0c44b0780d 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -56,6 +56,7 @@ dependencies { api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api group: 'com.google.code.gson', name: 'gson', version: '2.8.9' api group: 'com.tdunning', name: 't-digest', version: '3.3' + api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" api project(':common') testImplementation('org.junit.jupiter:junit-jupiter:5.9.3') diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java index c36cd3ea6d..20c74a7dc9 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java @@ -15,7 +15,12 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; +import java.util.ArrayList; +import java.util.List; import lombok.RequiredArgsConstructor; +import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.SemanticCheckException; @@ -25,6 +30,7 @@ public class ExprDateValue extends AbstractExprValue { private final LocalDate date; + private String dateFormat; /** Constructor of ExprDateValue. */ public ExprDateValue(String date) { @@ -36,9 +42,45 @@ public ExprDateValue(String date) { } } + /** Constructor of ExprDateValue to support custom/OpenSearch date formats in mappings. */ + public ExprDateValue(String date, List dateFormatters) { + LocalDate localDate = null; + String dateFormat = ""; + // check if dateFormatters are empty, then set default ones + if (dateFormatters == null || dateFormatters.isEmpty()) { + String defaultPatterns = + "strict_date_time_no_millis||strict_date_optional_time||epoch_millis"; + String[] patterns = defaultPatterns.split("\\|\\|"); + dateFormatters = new ArrayList<>(); + for (String pattern : patterns) { + dateFormatters.add(DateFormatter.forPattern(pattern)); + } + } + for (DateFormatter formatter : dateFormatters) { + try { + TemporalAccessor accessor = formatter.parse(date); + ZonedDateTime zonedDateTime = DateFormatters.from(accessor); + localDate = zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toLocalDate(); + + dateFormat = formatter.format(accessor); + break; + } catch (IllegalArgumentException ignored) { + // nothing to do, try another format + } + } + if (localDate == null) { + localDate = new ExprDateValue(date).dateValue(); + } + this.date = localDate; + this.dateFormat = dateFormat; + } + @Override public String value() { - return DateTimeFormatter.ISO_LOCAL_DATE.format(date); + if (this.dateFormat == null || this.dateFormat.isEmpty()) { + return DateTimeFormatter.ISO_LOCAL_DATE.format(date); + } + return this.dateFormat; } @Override @@ -68,7 +110,7 @@ public boolean isDateTime() { @Override public String toString() { - return String.format("DATE '%s'", value()); + return String.format("DATE '%s'", DateTimeFormatter.ISO_LOCAL_DATE.format(date)); } @Override diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java index 6b5a4a7c48..260f44a2a0 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java @@ -8,14 +8,15 @@ import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; +import java.time.*; import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import lombok.RequiredArgsConstructor; +import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.SemanticCheckException; @@ -26,6 +27,7 @@ public class ExprTimeValue extends AbstractExprValue { private final LocalTime time; + private String timeFormat; /** Constructor of ExprTimeValue. */ public ExprTimeValue(String time) { @@ -37,9 +39,44 @@ public ExprTimeValue(String time) { } } + /** Constructor of ExprTimeValue to support custom/OpenSearch date formats in mappings. */ + public ExprTimeValue(String time, List dateFormatters) { + LocalTime localTime = null; + String timeFormat = ""; + // check if dateFormatters are empty, then set default ones + if (dateFormatters == null || dateFormatters.isEmpty()) { + String defaultPatterns = + "strict_date_time_no_millis||strict_date_optional_time||epoch_millis"; + String[] patterns = defaultPatterns.split("\\|\\|"); + dateFormatters = new ArrayList<>(); + for (String pattern : patterns) { + dateFormatters.add(DateFormatter.forPattern(pattern)); + } + } + for (DateFormatter formatter : dateFormatters) { + try { + TemporalAccessor accessor = formatter.parse(time); + ZonedDateTime zonedDateTime = DateFormatters.from(accessor); + localTime = zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toLocalTime(); + timeFormat = formatter.format(accessor); + break; + } catch (IllegalArgumentException ignored) { + // nothing to do, try another format + } + } + if (localTime == null) { + localTime = new ExprTimeValue(time).timeValue(); + } + this.time = localTime; + this.timeFormat = timeFormat; + } + @Override public String value() { - return ISO_LOCAL_TIME.format(time); + if (this.timeFormat == null || this.timeFormat.isEmpty()) { + return ISO_LOCAL_TIME.format(time); + } + return this.timeFormat; } @Override diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java index e103dc7253..7b90782b38 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java @@ -5,18 +5,18 @@ package org.opensearch.sql.data.model; -import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS; -import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_WITHOUT_NANO; - -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.ZoneOffset; +import static org.opensearch.sql.utils.DateTimeFormatters.*; + +import java.time.*; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAccessor; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import lombok.RequiredArgsConstructor; +import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.SemanticCheckException; @@ -26,6 +26,7 @@ public class ExprTimestampValue extends AbstractExprValue { private final Instant timestamp; + private String dateTimeFormat; /** Constructor. */ public ExprTimestampValue(String timestamp) { @@ -42,6 +43,40 @@ public ExprTimestampValue(String timestamp) { } } + /** + * Constructor of ExprTimestampValue to support custom/OpenSearch dateTime formats in mappings. + */ + public ExprTimestampValue(String timestamp, List dateFormatters) { + Instant localDateTime = null; + String dateTimeFormat = ""; + // check if dateFormatters are empty, then set default ones + if (dateFormatters == null || dateFormatters.isEmpty()) { + String defaultPatterns = + "strict_date_time_no_millis||strict_date_optional_time||epoch_millis"; + String[] patterns = defaultPatterns.split("\\|\\|"); + dateFormatters = new ArrayList<>(); + for (String pattern : patterns) { + dateFormatters.add(DateFormatter.forPattern(pattern)); + } + } + for (DateFormatter formatter : dateFormatters) { + try { + TemporalAccessor accessor = formatter.parse(timestamp); + ZonedDateTime zonedDateTime = DateFormatters.from(accessor); + localDateTime = zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toInstant(); + dateTimeFormat = formatter.format(accessor); + break; + } catch (IllegalArgumentException ignored) { + // nothing to do, try another format + } + } + if (localDateTime == null) { + localDateTime = new ExprTimestampValue(timestamp).timestampValue(); + } + this.timestamp = localDateTime; + this.dateTimeFormat = dateTimeFormat; + } + /** localDateTime Constructor. */ public ExprTimestampValue(LocalDateTime localDateTime) { this.timestamp = localDateTime.atZone(ZoneOffset.UTC).toInstant(); @@ -49,11 +84,14 @@ public ExprTimestampValue(LocalDateTime localDateTime) { @Override public String value() { - return timestamp.getNano() == 0 - ? DATE_TIME_FORMATTER_WITHOUT_NANO - .withZone(ZoneOffset.UTC) - .format(timestamp.truncatedTo(ChronoUnit.SECONDS)) - : DATE_TIME_FORMATTER_VARIABLE_NANOS.withZone(ZoneOffset.UTC).format(timestamp); + if (this.dateTimeFormat == null || this.dateTimeFormat.isEmpty()) { + return timestamp.getNano() == 0 + ? DATE_TIME_FORMATTER_WITHOUT_NANO + .withZone(ZoneOffset.UTC) + .format(timestamp.truncatedTo(ChronoUnit.SECONDS)) + : DATE_TIME_FORMATTER_VARIABLE_NANOS.withZone(ZoneOffset.UTC).format(timestamp); + } + return this.dateTimeFormat; } @Override diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java index 18e6541514..8dd8f2ccb7 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java @@ -130,7 +130,8 @@ public class DateTimeFormatters { public static final DateTimeFormatter DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL = new DateTimeFormatterBuilder() - .appendPattern("[uuuu-MM-dd HH:mm:ss][uuuu-MM-dd HH:mm][HH:mm:ss][HH:mm][uuuu-MM-dd]") + .appendPattern( + "[uuuu-MM-dd HH:mm:ss][uuuu-MM-dd HH:mm][HH:mm:ss][HH:mm][uuuu-MM-dd][dd-MMM-uu]") .appendFraction( ChronoField.NANO_OF_SECOND, MIN_FRACTION_SECONDS, MAX_FRACTION_SECONDS, true) .toFormatter(Locale.ROOT) diff --git a/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java index b5a3d61211..8b36e7d55a 100644 --- a/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java @@ -5,18 +5,18 @@ package org.opensearch.sql.data.model; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.*; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import org.junit.jupiter.api.Test; +import org.opensearch.common.time.DateFormatter; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.FunctionProperties; @@ -96,11 +96,265 @@ public void dateValueInterfaceTest() { } @Test - public void dateInUnsupportedFormat() { - SemanticCheckException exception = - assertThrows(SemanticCheckException.class, () -> new ExprDateValue("2020-07-07Z")); + void testValidTimestampWithCustomFormatter() { + String timestamp = "2021-11-08T17:00:00Z"; + DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ssX"); + ExprTimestampValue timestampValue = + new ExprTimestampValue(timestamp, Collections.singletonList(formatter)); + + assertEquals("2021-11-08T17:00:00Z", timestampValue.value()); + assertEquals(LocalDate.parse("2021-11-08"), timestampValue.dateValue()); + assertEquals(LocalTime.parse("17:00:00"), timestampValue.timeValue()); + assertEquals(TIMESTAMP, timestampValue.type()); assertEquals( - "date:2020-07-07Z in unsupported format, please use 'yyyy-MM-dd'", exception.getMessage()); + ZonedDateTime.of(LocalDateTime.parse("2021-11-08T17:00:00"), ZoneOffset.UTC).toInstant(), + timestampValue.timestampValue()); + assertEquals("TIMESTAMP '2021-11-08T17:00:00Z'", timestampValue.toString()); + assertEquals( + LocalDateTime.parse("2021-11-08T17:00:00"), + LocalDateTime.ofInstant(timestampValue.timestampValue(), ZoneOffset.UTC)); + assertThrows( + ExpressionEvaluationException.class, + () -> integerValue(1).timestampValue(), + "invalid to get timestampValue from value of type INTEGER"); + } + + @Test + void testValidTimestampWithMultipleFormatters() { + String timestamp = "2021-11-08T17:00:00Z"; + DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd'T'HH:mm:ssX"); + DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ssX"); + + ExprTimestampValue timestampValue = + new ExprTimestampValue(timestamp, Arrays.asList(formatter1, formatter2)); + + assertEquals("2021-11-08T17:00:00Z", timestampValue.value()); + assertEquals(LocalDate.parse("2021-11-08"), timestampValue.dateValue()); + assertEquals(LocalTime.parse("17:00:00"), timestampValue.timeValue()); + assertEquals(TIMESTAMP, timestampValue.type()); + + String timestamp2 = "2021/11/08T17:00:00Z"; + + ExprTimestampValue timestampValue2 = + new ExprTimestampValue(timestamp2, Arrays.asList(formatter1, formatter2)); + + assertEquals("2021/11/08T17:00:00Z", timestampValue2.value()); + assertEquals(LocalDate.parse("2021-11-08"), timestampValue2.dateValue()); + assertEquals(LocalTime.parse("17:00:00"), timestampValue2.timeValue()); + assertEquals(TIMESTAMP, timestampValue2.type()); + } + + @Test + void testEmptyFormatterListForTimeStamp() { + String timestamp = "2021-11-08T17:00:00"; + Instant expectedDateTime = + ZonedDateTime.of(2021, 11, 8, 17, 0, 0, 0, ZoneOffset.UTC).toInstant(); + + // Test with null formatter list + ExprTimestampValue valueWithNullFormatter = new ExprTimestampValue(timestamp, null); + assertEquals(expectedDateTime, valueWithNullFormatter.timestampValue()); + assertEquals("2021-11-08T17:00:00.000Z", valueWithNullFormatter.value()); + + // Test with empty formatter list + ExprTimestampValue valueWithEmptyFormatter = + new ExprTimestampValue(timestamp, Collections.emptyList()); + assertEquals(expectedDateTime, valueWithEmptyFormatter.timestampValue()); + assertEquals("2021-11-08T17:00:00.000Z", valueWithEmptyFormatter.value()); + } + + @Test + void testOpenSearchDateTimeNamedFormatter() { + String timestamp = "2019-03-23T21:34:46"; + DateFormatter formatter = DateFormatter.forPattern("strict_date_hour_minute_second"); + ExprTimestampValue value = + new ExprTimestampValue(timestamp, Collections.singletonList(formatter)); + + assertEquals("2019-03-23T21:34:46", value.value()); + assertEquals(LocalDate.parse("2019-03-23"), value.dateValue()); + assertEquals(LocalTime.parse("21:34:46"), value.timeValue()); + assertEquals(TIMESTAMP, value.type()); + } + + @Test + void testInvalidTimestamp() { + String timestamp = "invalid-timestamp"; + DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd'T'HH:mm:ssX"); + DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ssX"); + + List formatters = Arrays.asList(formatter1, formatter2); + + try { + new ExprTimestampValue(timestamp, formatters); + } catch (SemanticCheckException e) { + assertEquals( + String.format( + "timestamp:%s in unsupported format, please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'", + timestamp), + e.getMessage()); + } + } + + @Test + void testEpochDateTimeFormatter() { + long epochTimestamp = 1636390800000L; // Corresponds to "2021-11-08T17:00:00Z" + DateFormatter formatter = DateFormatter.forPattern("epoch_millis"); + + ExprTimestampValue value = + new ExprTimestampValue(Long.toString(epochTimestamp), Collections.singletonList(formatter)); + assertEquals("1636390800000", value.value()); + assertEquals(LocalDate.parse("2021-11-08"), value.dateValue()); + assertEquals(LocalTime.parse("17:00:00"), value.timeValue()); + assertEquals(TIMESTAMP, value.type()); + } + + @Test + void testValidDateWithCustomFormatter() { + String dateString = "2021-11-08"; + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ISO_DATE); + DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd"); + ExprDateValue value = new ExprDateValue(dateString, Collections.singletonList(formatter)); + assertEquals(expectedDate, value.dateValue()); + assertEquals("2021-11-08", value.value()); + } + + @Test + void testValidDateWithMultipleFormatters() { + String dateString = "2021-11-08"; + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ofPattern("yyyy-MM-dd")); + DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd"); + DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd"); + + ExprDateValue value = new ExprDateValue(dateString, List.of(formatter1, formatter2)); + + assertEquals(expectedDate, value.dateValue()); + assertEquals("2021-11-08", value.value()); + } + + @Test + void testInvalidDate() { + String dateString = "invalid-date"; + DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd"); + DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd"); + + try { + new ExprDateValue(dateString, List.of(formatter1, formatter2)); + } catch (SemanticCheckException e) { + assertEquals( + String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", dateString), + e.getMessage()); + } + } + + @Test + void testEmptyFormatterListForDate() { + String dateString = "2021-11-08"; + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ISO_LOCAL_DATE); + + // Test with null formatter list + ExprDateValue valueWithNullFormatter = new ExprDateValue(dateString, null); + assertEquals(expectedDate, valueWithNullFormatter.dateValue()); + // Formatted to default strict_date_optional_time format + assertEquals("2021-11-08T00:00:00.000Z", valueWithNullFormatter.value()); + + // Test with empty formatter list + ExprDateValue valueWithEmptyFormatter = new ExprDateValue(dateString, Collections.emptyList()); + assertEquals(expectedDate, valueWithEmptyFormatter.dateValue()); + // Formatted to default strict_date_optional_time format + assertEquals("2021-11-08T00:00:00.000Z", valueWithEmptyFormatter.value()); + } + + @Test + void testValidTimeWithCustomFormatter() { + String timeString = "12:10:30.000"; + LocalTime expectedTime = + LocalTime.parse(timeString, DateTimeFormatter.ofPattern("HH:mm:ss.SSS")); + DateFormatter formatter = DateFormatter.forPattern("HH:mm:ss.SSS"); + + ExprTimeValue value = new ExprTimeValue(timeString, Collections.singletonList(formatter)); + + assertEquals(expectedTime, value.timeValue()); + assertEquals("12:10:30.000", value.value()); + assertEquals("TIME '12:10:30.000'", value.toString()); + } + + @Test + void testValidTimeWithMultipleFormatters() { + String timeString = "12:10:30"; + LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ofPattern("HH:mm:ss")); + DateFormatter formatter1 = DateFormatter.forPattern("HH:mm:ss.SSS"); + DateFormatter formatter2 = DateFormatter.forPattern("HH:mm:ss"); + + ExprTimeValue value = new ExprTimeValue(timeString, List.of(formatter1, formatter2)); + + assertEquals(expectedTime, value.timeValue()); + assertEquals("12:10:30", value.value()); + } + + @Test + void testInvalidTime() { + String timeString = "invalid-time"; + DateFormatter formatter1 = DateFormatter.forPattern("HH:mm:ss.SSS"); + DateFormatter formatter2 = DateFormatter.forPattern("HH:mm:ss"); + + try { + new ExprTimeValue(timeString, List.of(formatter1, formatter2)); + } catch (SemanticCheckException e) { + assertEquals( + String.format( + "time:%s in unsupported format, please use 'HH:mm:ss[.SSSSSSSSS]'", timeString), + e.getMessage()); + } + } + + @Test + void testEmptyFormatterListForTime() { + String timeString = "12:10:30"; + LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ISO_LOCAL_TIME); + + // Test with null formatter list + ExprTimeValue valueWithNullFormatter = new ExprTimeValue(timeString, null); + assertEquals(expectedTime, valueWithNullFormatter.timeValue()); + assertEquals("12:10:30", valueWithNullFormatter.value()); + + // Test with empty formatter list + ExprTimeValue valueWithEmptyFormatter = new ExprTimeValue(timeString, Collections.emptyList()); + assertEquals(expectedTime, valueWithEmptyFormatter.timeValue()); + assertEquals("12:10:30", valueWithEmptyFormatter.value()); + } + + @Test + void testEpochTimeFormatter() { + long epochMilli = 1420070400000L; // epoch time in milliseconds + LocalTime expectedTime = + ZonedDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC).toLocalTime(); + DateFormatter epochFormatter = DateFormatter.forPattern("epoch_millis"); + + ExprTimeValue value = + new ExprTimeValue(String.valueOf(epochMilli), Collections.singletonList(epochFormatter)); + + assertEquals(expectedTime, value.timeValue()); + assertEquals(String.valueOf(epochMilli), value.value()); + assertEquals(TIME, value.type()); + assertTrue(value.isDateTime()); + assertEquals("TIME '1420070400000'", value.toString()); + + var exception = assertThrows(ExpressionEvaluationException.class, value::dateValue); + assertEquals("invalid to get dateValue from value of type TIME", exception.getMessage()); + exception = assertThrows(ExpressionEvaluationException.class, value::timestampValue); + assertEquals("invalid to get timestampValue from value of type TIME", exception.getMessage()); + exception = + assertThrows(ExpressionEvaluationException.class, () -> integerValue(1).timeValue()); + assertEquals("invalid to get timeValue from value of type INTEGER", exception.getMessage()); + + var functionProperties = new FunctionProperties(); + var today = LocalDate.now(functionProperties.getQueryStartClock()); + assertEquals(today, value.dateValue(functionProperties)); + assertEquals( + today.atTime(0, 0, 0), + LocalDateTime.ofInstant(value.timestampValue(functionProperties), ZoneOffset.UTC)); + assertEquals( + ZonedDateTime.of(LocalTime.parse("00:00:00").atDate(today), ZoneOffset.UTC).toInstant(), + value.timestampValue(functionProperties)); } @Test diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index c820c97196..775881f547 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1238,20 +1238,17 @@ public void testWeekOfYearWithTimeType() { DSL.week( functionProperties, DSL.literal(new ExprTimeValue("12:23:34")), DSL.literal(0)), "week(TIME '12:23:34', 0)", - LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR) - - 1), + LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR)), () -> validateStringFormat( DSL.week_of_year(functionProperties, DSL.literal(new ExprTimeValue("12:23:34"))), "week_of_year(TIME '12:23:34')", - LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR) - - 1), + LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR)), () -> validateStringFormat( DSL.weekofyear(functionProperties, DSL.literal(new ExprTimeValue("12:23:34"))), "weekofyear(TIME '12:23:34')", - LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR) - - 1)); + LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR))); } @Test diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 02d50d0b59..4d6e6e5789 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -5,7 +5,6 @@ package org.opensearch.sql.expression.datetime; -import static java.time.temporal.ChronoField.ALIGNED_WEEK_OF_YEAR; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.opensearch.sql.data.type.ExprCoreType.LONG; @@ -94,11 +93,11 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { public void testExtractDatePartWithTimeType() { datePartWithTimeArgQuery( "DAY", timeInput, LocalDate.now(functionProperties.getQueryStartClock()).getDayOfMonth()); - + /* datePartWithTimeArgQuery( "WEEK", timeInput, - LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR)); + LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR));**/ datePartWithTimeArgQuery( "MONTH", timeInput, LocalDate.now(functionProperties.getQueryStartClock()).getMonthValue()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 47225ac601..33f915b71a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -101,7 +101,7 @@ public void testYearweekWithoutMode() { // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test public void testYearweekWithTimeType() { - int week = LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR) - 1; + int week = LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR); int year = LocalDate.now(functionProperties.getQueryStartClock()).getYear(); int expected = Integer.parseInt(String.format("%d%02d", year, week)); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index ddbba61260..2c38927ba1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -62,7 +62,7 @@ public String toString() { @EqualsAndHashCode.Exclude @Getter protected MappingType mappingType; // resolved ExprCoreType - protected ExprCoreType exprCoreType; + @Getter protected ExprCoreType exprCoreType; /** * Get a simplified type {@link ExprCoreType} if possible. To avoid returning `UNKNOWN` for @@ -71,10 +71,12 @@ public String toString() { * @return An {@link ExprType}. */ public ExprType getExprType() { - if (exprCoreType != ExprCoreType.UNKNOWN) { - return exprCoreType; - } - return this; + return (exprCoreType == ExprCoreType.DATE + || exprCoreType == ExprCoreType.TIME + || exprCoreType == ExprCoreType.TIMESTAMP + || exprCoreType == ExprCoreType.UNKNOWN) + ? this + : exprCoreType; } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index 7e6bee77c2..d6dd9d7c50 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -164,6 +164,10 @@ public boolean hasFormats() { return !formats.isEmpty(); } + public List getFormats() { + return this.formats; + } + /** * Retrieves and splits a user defined format string from the mapping into a list of formats. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 3341e01ab2..3cb182de5b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -230,7 +230,7 @@ private Optional type(String field) { private static ExprValue parseDateTimeString(String value, OpenSearchDateType dataType) { List formatters = dataType.getAllNamedFormatters(); formatters.addAll(dataType.getAllCustomFormatters()); - ExprCoreType returnFormat = (ExprCoreType) dataType.getExprType(); + ExprCoreType returnFormat = dataType.getExprCoreType(); for (DateFormatter formatter : formatters) { try { @@ -273,8 +273,7 @@ private static ExprValue parseDateTimeString(String value, OpenSearchDateType da private static ExprValue createOpenSearchDateType(Content value, ExprType type) { OpenSearchDateType dt = (OpenSearchDateType) type; - ExprType returnFormat = dt.getExprType(); - + ExprCoreType returnFormat = dt.getExprCoreType(); if (value.isNumber()) { // isNumber var numFormatters = dt.getNumericNamedFormatters(); if (numFormatters.size() > 0 || !dt.hasFormats()) { @@ -287,7 +286,7 @@ private static ExprValue createOpenSearchDateType(Content value, ExprType type) epochMillis = value.longValue(); } Instant instant = Instant.ofEpochMilli(epochMillis); - switch ((ExprCoreType) returnFormat) { + switch (returnFormat) { case TIME: return new ExprTimeValue(LocalTime.from(instant.atZone(ZoneOffset.UTC))); case DATE: diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index c6afdb8511..e3fa4d4efe 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -118,7 +118,9 @@ public Map getFieldTypes() { OpenSearchDataType.traverseAndFlatten(cachedFieldOpenSearchTypes).entrySet().stream() .collect( LinkedHashMap::new, - (map, item) -> map.put(item.getKey(), item.getValue().getExprType()), + (map, item) -> { + map.put(item.getKey(), item.getValue().getExprType()); + }, Map::putAll); } return cachedFieldTypes; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java index 11533c754e..92e914b12c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java @@ -8,8 +8,10 @@ import static org.opensearch.sql.analysis.NestedAnalyzer.isNestedFunction; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Map; import java.util.function.Function; +import org.opensearch.common.time.DateFormatter; import org.opensearch.index.query.QueryBuilder; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; @@ -32,10 +34,13 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; /** Lucene query abstraction that builds Lucene query from function expression. */ public abstract class LuceneQuery { + private List dateFormatters; + /** * Check if function expression supported by current Lucene query. Default behavior is that report * supported if: @@ -103,9 +108,15 @@ private boolean literalExpressionWrappedByCast(FunctionExpression func) { */ public QueryBuilder build(FunctionExpression func) { ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0); + if (ref.type() instanceof OpenSearchDateType) { + OpenSearchDateType openSearchDateType = (OpenSearchDateType) ref.type(); + dateFormatters = openSearchDateType.getAllNamedFormatters(); + dateFormatters.addAll(openSearchDateType.getAllCustomFormatters()); + } Expression expr = func.getArguments().get(1); ExprValue literalValue = expr instanceof LiteralExpression ? expr.valueOf() : cast((FunctionExpression) expr); + return doBuild(ref.getAttr(), ref.type(), literalValue); } @@ -120,7 +131,7 @@ private ExprValue cast(FunctionExpression castFunction) { ImmutableMap.>builder() .put( BuiltinFunctionName.CAST_TO_STRING.getName(), - expr -> { + (expr) -> { if (!expr.type().equals(ExprCoreType.STRING)) { return new ExprStringValue(String.valueOf(expr.valueOf().value())); } else { @@ -210,7 +221,7 @@ private ExprValue cast(FunctionExpression castFunction) { BuiltinFunctionName.CAST_TO_DATE.getName(), expr -> { if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprDateValue(expr.valueOf().stringValue()); + return new ExprDateValue(expr.valueOf().stringValue(), dateFormatters); } else { return new ExprDateValue(expr.valueOf().dateValue()); } @@ -219,7 +230,7 @@ private ExprValue cast(FunctionExpression castFunction) { BuiltinFunctionName.CAST_TO_TIME.getName(), expr -> { if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprTimeValue(expr.valueOf().stringValue()); + return new ExprTimeValue(expr.valueOf().stringValue(), dateFormatters); } else { return new ExprTimeValue(expr.valueOf().timeValue()); } @@ -228,7 +239,7 @@ private ExprValue cast(FunctionExpression castFunction) { BuiltinFunctionName.CAST_TO_TIMESTAMP.getName(), expr -> { if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprTimestampValue(expr.valueOf().stringValue()); + return new ExprTimestampValue(expr.valueOf().stringValue(), dateFormatters); } else { return new ExprTimestampValue(expr.valueOf().timestampValue()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java index 2e33e3cc7c..6ba02eb6f0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java @@ -5,6 +5,7 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene; +import java.util.List; import lombok.RequiredArgsConstructor; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -12,6 +13,7 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; /** Lucene query that builds range query for non-quality comparison. */ @RequiredArgsConstructor @@ -28,8 +30,14 @@ public enum Comparison { /** Comparison that range query build for. */ private final Comparison comparison; + List fieldFormats; + @Override protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) { + if (fieldType instanceof OpenSearchDateType) { + OpenSearchDateType openSearchDateType = (OpenSearchDateType) fieldType; + fieldFormats = openSearchDateType.getFormats(); + } Object value = value(literal); RangeQueryBuilder query = QueryBuilders.rangeQuery(fieldName); @@ -48,7 +56,8 @@ protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue l } private Object value(ExprValue literal) { - if (literal.type().equals(ExprCoreType.TIMESTAMP)) { + if (literal.type().equals(ExprCoreType.TIMESTAMP) + && (fieldFormats == null || fieldFormats.isEmpty())) { return literal.timestampValue().toEpochMilli(); } else { return literal.value(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java index cd506898d7..a93a40e0b2 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java @@ -5,24 +5,32 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene; +import java.util.List; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; /** Lucene query that build term query for equality comparison. */ public class TermQuery extends LuceneQuery { + List fieldFormats; @Override protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) { + if (fieldType instanceof OpenSearchDateType) { + OpenSearchDateType openSearchDateType = (OpenSearchDateType) fieldType; + fieldFormats = openSearchDateType.getFormats(); + } fieldName = OpenSearchTextType.convertTextToKeyword(fieldName, fieldType); return QueryBuilders.termQuery(fieldName, value(literal)); } private Object value(ExprValue literal) { - if (literal.type().equals(ExprCoreType.TIMESTAMP)) { + if (literal.type().equals(ExprCoreType.TIMESTAMP) + && (fieldFormats == null || fieldFormats.isEmpty())) { return literal.timestampValue().toEpochMilli(); } else { return literal.value(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 82e6222dc4..f7b60b8a35 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -124,7 +124,13 @@ public void of_MappingType(MappingType mappingType, String name, ExprType dataTy assertAll( () -> assertEquals(nameForPPL, type.typeName()), () -> assertEquals(nameForSQL, type.legacyTypeName()), - () -> assertEquals(dataType, type.getExprType())); + () -> { + if (dataType == ExprCoreType.TIMESTAMP || dataType == ExprCoreType.DATE) { + assertEquals(dataType, type.getExprCoreType()); // Assuming the expected type is DATE + } else { + assertEquals(dataType, type.getExprType()); + } + }); } @ParameterizedTest(name = "{0}") @@ -133,7 +139,7 @@ public void of_ExprCoreType(ExprCoreType coreType) { assumeFalse(coreType == UNKNOWN); var type = OpenSearchDataType.of(coreType); if (type instanceof OpenSearchDateType) { - assertEquals(coreType, type.getExprType()); + assertEquals(coreType, type.getExprCoreType()); } else { assertEquals(coreType.toString(), type.typeName()); assertEquals(coreType.toString(), type.legacyTypeName()); @@ -416,7 +422,7 @@ public void test_getExprType() { assertEquals(FLOAT, OpenSearchDataType.of(MappingType.HalfFloat).getExprType()); assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.Double).getExprType()); assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.ScaledFloat).getExprType()); - assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprType()); + assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprCoreType()); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java index c6885c8ffe..6a878eb024 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java @@ -94,9 +94,9 @@ public void check_legacyTypeName() { public void check_exprTypeName() { assertAll( // exprType changes based on type (no datetime): - () -> assertEquals(TIMESTAMP, defaultDateType.getExprType()), - () -> assertEquals(TIME, timeDateType.getExprType()), - () -> assertEquals(DATE, dateDateType.getExprType())); + () -> assertEquals(TIMESTAMP, defaultDateType.getExprCoreType()), + () -> assertEquals(TIME, timeDateType.getExprCoreType()), + () -> assertEquals(DATE, dateDateType.getExprCoreType())); } private static Stream getAllSupportedFormats() { @@ -129,22 +129,22 @@ public void check_datetime_format_names(FormatNames datetimeFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIMESTAMP, camelCaseName + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); + + dateType.getExprCoreType()); } String snakeCaseName = datetimeFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIMESTAMP, snakeCaseName + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); + + dateType.getExprCoreType()); } else { fail(); } @@ -161,18 +161,22 @@ public void check_date_format_names(FormatNames dateFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), DATE, - camelCaseName + " does not format to a DATE type, instead got " + dateType.getExprType()); + camelCaseName + + " does not format to a DATE type, instead got " + + dateType.getExprCoreType()); } String snakeCaseName = dateFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), DATE, - snakeCaseName + " does not format to a DATE type, instead got " + dateType.getExprType()); + snakeCaseName + + " does not format to a DATE type, instead got " + + dateType.getExprCoreType()); } else { fail(); } @@ -189,18 +193,22 @@ public void check_time_format_names(FormatNames timeFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIME, - camelCaseName + " does not format to a TIME type, instead got " + dateType.getExprType()); + camelCaseName + + " does not format to a TIME type, instead got " + + dateType.getExprCoreType()); } String snakeCaseName = timeFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIME, - snakeCaseName + " does not format to a TIME type, instead got " + dateType.getExprType()); + snakeCaseName + + " does not format to a TIME type, instead got " + + dateType.getExprCoreType()); } else { fail(); } @@ -244,9 +252,9 @@ private static Stream get_format_combinations_for_test() { @MethodSource("get_format_combinations_for_test") public void check_ExprCoreType_of_combinations_of_custom_and_predefined_formats( ExprCoreType expected, List formats, String testName) { - assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprCoreType()); formats = Lists.reverse(formats); - assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprCoreType()); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index 3ddb07d86a..3ca566fac6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -148,7 +148,7 @@ void getFieldTypes() { hasEntry("gender", ExprCoreType.BOOLEAN), hasEntry("family", ExprCoreType.ARRAY), hasEntry("employer", ExprCoreType.STRUCT), - hasEntry("birthday", ExprCoreType.TIMESTAMP), + hasEntry("birthday", (ExprType) OpenSearchDataType.of(MappingType.Date)), hasEntry("id1", ExprCoreType.BYTE), hasEntry("id2", ExprCoreType.SHORT), hasEntry("blob", (ExprType) OpenSearchDataType.of(MappingType.Binary)))); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index 90b982e017..53b228622c 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -1767,7 +1767,7 @@ void cast_to_date_in_filter() { "{\n" + " \"term\" : {\n" + " \"date_value\" : {\n" - + " \"value\" : \"2021-11-08\",\n" + + " \"value\" : \"2021-11-08T00:00:00.000Z\",\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" @@ -1775,6 +1775,15 @@ void cast_to_date_in_filter() { assertJsonEquals( json, buildQuery(DSL.equal(ref("date_value", DATE), DSL.castDate(literal("2021-11-08"))))); + json = + "{\n" + + " \"term\" : {\n" + + " \"date_value\" : {\n" + + " \"value\" : \"2021-11-08\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}"; assertJsonEquals( json, buildQuery(