From 93760bc121974e63d1bee5f1686e8b754c15f789 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Tue, 1 Oct 2024 00:48:33 +0530 Subject: [PATCH] addressing comments Signed-off-by: Bharathwaj G --- .../index/mapper/StarTreeMapperIT.java | 12 +-- .../datacube/DateDimension.java | 25 +++--- .../compositeindex/datacube/Dimension.java | 11 ++- .../datacube/NumericDimension.java | 8 +- .../datacube/ReadDimension.java | 8 +- .../datacube/startree/StarTreeField.java | 2 +- .../startree/StarTreeIndexSettings.java | 8 +- .../startree/builder/BaseStarTreeBuilder.java | 11 ++- ...imeUnit.java => DataCubeDateTimeUnit.java} | 16 ++-- .../datacube/startree/DateDimensionTests.java | 82 +++++++++++-------- .../datacube/startree/StarTreeTestUtils.java | 2 +- .../builder/AbstractStarTreeBuilderTests.java | 18 ++-- .../index/mapper/StarTreeMapperTests.java | 10 +-- 13 files changed, 119 insertions(+), 94 deletions(-) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/{ExtendedDateTimeUnit.java => DataCubeDateTimeUnit.java} (81%) diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index af0317ba31562..bbaebfc15456e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -27,9 +27,9 @@ import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DataCubeDateTimeUnit; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; -import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit; import org.opensearch.index.query.QueryBuilders; import org.opensearch.indices.IndicesService; import org.opensearch.search.SearchHit; @@ -392,7 +392,7 @@ public void testValidCompositeIndex() { DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); List expectedTimeUnits = Arrays.asList( new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR), - ExtendedDateTimeUnit.HALF_HOUR_OF_DAY + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY ); for (int i = 0; i < dateDim.getSortedCalendarIntervals().size(); i++) { assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); @@ -428,8 +428,8 @@ public void testValidCompositeIndexWithDates() { assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); List expectedTimeUnits = Arrays.asList( - ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY, - ExtendedDateTimeUnit.HALF_HOUR_OF_DAY, + DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY, + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY, new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH) ); for (int i = 0; i < dateDim.getIntervals().size(); i++) { @@ -466,7 +466,7 @@ public void testValidCompositeIndexWithDuplicateDates() { assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); List expectedTimeUnits = Arrays.asList( - ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY, + DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY, new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH) ); for (int i = 0; i < dateDim.getIntervals().size(); i++) { @@ -637,7 +637,7 @@ public void testUpdateIndexWhenMappingIsSame() { DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); List expectedTimeUnits = Arrays.asList( new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR), - ExtendedDateTimeUnit.HALF_HOUR_OF_DAY + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY ); for (int i = 0; i < expectedTimeUnits.size(); i++) { assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getIntervals().get(i).shortName()); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java index 00e6162afa16e..53ca12a613d05 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java @@ -12,8 +12,8 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.time.DateUtils; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DataCubeDateTimeUnit; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; -import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit; import org.opensearch.index.mapper.CompositeDataCubeFieldType; import org.opensearch.index.mapper.DateFieldMapper; @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Consumer; import java.util.stream.Collectors; /** @@ -64,26 +65,24 @@ public List getSortedCalendarIntervals() { * Sets the dimension values in sorted order in the provided array starting from the given index. * * @param val The value to be set - * @param dims The dimensions array to set the values in - * @param index The starting index in the array - * @return The next available index in the array + * @param dimSetter Consumer which sets the dimensions */ @Override - public int setDimensionValues(final Long val, final Long[] dims, int index) { + public void setDimensionValues(final Long val, final Consumer dimSetter) { for (DateTimeUnitRounding dateTimeUnit : sortedCalendarIntervals) { if (val == null) { - dims[index++] = null; - continue; + dimSetter.accept(null); + } else { + Long roundedValue = dateTimeUnit.roundFloor(storedDurationSinceEpoch(val)); + dimSetter.accept(roundedValue); } - dims[index++] = dateTimeUnit.roundFloor(maybeConvertNanosToMillis(val)); } - return index; } /** * Converts nanoseconds to milliseconds based on the resolution of the field */ - private long maybeConvertNanosToMillis(long nanoSecondsSinceEpoch) { + private long storedDurationSinceEpoch(long nanoSecondsSinceEpoch) { if (resolution.equals(DateFieldMapper.Resolution.NANOSECONDS)) return DateUtils.toMilliSeconds(nanoSecondsSinceEpoch); return nanoSecondsSinceEpoch; } @@ -92,7 +91,7 @@ private long maybeConvertNanosToMillis(long nanoSecondsSinceEpoch) { * Returns the list of fields that represent the dimension */ @Override - public List getDimensionFieldsNames() { + public List getSubDimensionNames() { List fields = new ArrayList<>(calendarIntervals.size()); for (DateTimeUnitRounding interval : sortedCalendarIntervals) { // TODO : revisit this post file format changes @@ -147,8 +146,8 @@ public static class DateTimeUnitComparator implements Comparator dimSetter); /** * Returns the list of dimension fields that represent the dimension */ - List getDimensionFieldsNames(); + List getSubDimensionNames(); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java index d88b6aac5c171..e8fabf49b5e5b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.util.List; import java.util.Objects; +import java.util.function.Consumer; /** * Composite index numeric dimension class @@ -40,13 +41,12 @@ public int getNumSubDimensions() { } @Override - public int setDimensionValues(final Long val, final Long[] dims, int index) { - dims[index++] = val; - return index; + public void setDimensionValues(final Long val, final Consumer dimSetter) { + dimSetter.accept(val); } @Override - public List getDimensionFieldsNames() { + public List getSubDimensionNames() { // TODO : revisit this post file format changes return List.of(field); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java index 0d67f5ade73a4..be3667f10b6da 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.util.List; import java.util.Objects; +import java.util.function.Consumer; /** * Represents a dimension for reconstructing StarTreeField from file formats during searches and merges. @@ -38,13 +39,12 @@ public int getNumSubDimensions() { } @Override - public int setDimensionValues(Long value, Long[] dims, int index) { - dims[index++] = value; - return index; + public void setDimensionValues(final Long val, final Consumer dimSetter) { + dimSetter.accept(val); } @Override - public List getDimensionFieldsNames() { + public List getSubDimensionNames() { return List.of(field); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java index 36e6a9c539f42..1346186abfe44 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java @@ -42,7 +42,7 @@ public StarTreeField(String name, List dimensions, List metri this.starTreeConfig = starTreeConfig; dimensionNames = new ArrayList<>(); for (Dimension dimension : dimensions) { - dimensionNames.addAll(dimension.getDimensionFieldsNames()); + dimensionNames.addAll(dimension.getSubDimensionNames()); } metricNames = new ArrayList<>(); for (Metric metric : metrics) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java index b33c59e30ba74..13064b5b69f44 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java @@ -11,9 +11,9 @@ import org.opensearch.common.Rounding; import org.opensearch.common.settings.Setting; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DataCubeDateTimeUnit; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; -import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import java.util.Arrays; @@ -102,7 +102,7 @@ public class StarTreeIndexSettings { */ public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( "index.composite_index.star_tree.field.default.date_intervals", - Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), ExtendedDateTimeUnit.HALF_HOUR_OF_DAY.shortName()), + Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), DataCubeDateTimeUnit.HALF_HOUR_OF_DAY.shortName()), StarTreeIndexSettings::getTimeUnit, Setting.Property.IndexScope, Setting.Property.Final @@ -122,8 +122,8 @@ public class StarTreeIndexSettings { public static DateTimeUnitRounding getTimeUnit(String expression) { if (DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { return new DateTimeUnitAdapter(DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression)); - } else if (ExtendedDateTimeUnit.DATE_FIELD_UNITS.containsKey(expression)) { - return ExtendedDateTimeUnit.DATE_FIELD_UNITS.get(expression); + } else if (DataCubeDateTimeUnit.DATE_FIELD_UNITS.containsKey(expression)) { + return DataCubeDateTimeUnit.DATE_FIELD_UNITS.get(expression); } throw new IllegalArgumentException("unknown calendar intervals specified in star tree index mapping"); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index d1bb18681c5b8..c3b58c59720d2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -325,7 +325,7 @@ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, A FieldInfo[] metricFieldInfoList = new FieldInfo[metricAggregatorInfos.size()]; int dimIndex = 0; for (Dimension dim : dimensionsSplitOrder) { - for (String name : dim.getDimensionFieldsNames()) { + for (String name : dim.getSubDimensionNames()) { final FieldInfo fi = getFieldInfo( fullyQualifiedFieldNameForStarTreeDimensionsDocValues(starTreeField.getName(), name), DocValuesType.SORTED_NUMERIC, @@ -539,7 +539,7 @@ protected StarTreeDocument getSegmentStarTreeDocument( */ Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIterator[] dimensionReaders) throws IOException { Long[] dimensions = new Long[numDimensions]; - int dimIndex = 0; + AtomicInteger dimIndex = new AtomicInteger(0); for (int i = 0; i < dimensionReaders.length; i++) { if (dimensionReaders[i] != null) { try { @@ -551,11 +551,16 @@ Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIte logger.error("unable to read the dimension values from the segment", e); throw new IllegalStateException("unable to read the dimension values from the segment", e); } - dimIndex = dimensionsSplitOrder.get(i).setDimensionValues(dimensionReaders[i].value(currentDocId), dimensions, dimIndex); + dimensionsSplitOrder.get(i).setDimensionValues(dimensionReaders[i].value(currentDocId), value -> { + dimensions[dimIndex.getAndIncrement()] = value; + }); } else { throw new IllegalStateException("dimension readers are empty"); } } + if (dimIndex.get() == numDimensions) { + throw new IllegalStateException("Values are not set for all dimensions"); + } return dimensions; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/ExtendedDateTimeUnit.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DataCubeDateTimeUnit.java similarity index 81% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/ExtendedDateTimeUnit.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DataCubeDateTimeUnit.java index d8bab994e5aaa..5fd827df36ccb 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/ExtendedDateTimeUnit.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DataCubeDateTimeUnit.java @@ -28,7 +28,7 @@ * * @opensearch.experimental */ -public enum ExtendedDateTimeUnit implements DateTimeUnitRounding { +public enum DataCubeDateTimeUnit implements DateTimeUnitRounding { HALF_HOUR_OF_DAY("half-hour") { @Override public long roundFloor(long utcMillis) { @@ -42,19 +42,19 @@ public long roundFloor(long utcMillis) { } }; - public static final Map DATE_FIELD_UNITS; + public static final Map DATE_FIELD_UNITS; static { - Map dateFieldUnits = new HashMap<>(); - dateFieldUnits.put("30m", ExtendedDateTimeUnit.HALF_HOUR_OF_DAY); - dateFieldUnits.put("half-hour", ExtendedDateTimeUnit.HALF_HOUR_OF_DAY); - dateFieldUnits.put("15m", ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY); - dateFieldUnits.put("quarter-hour", ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY); + Map dateFieldUnits = new HashMap<>(); + dateFieldUnits.put("30m", DataCubeDateTimeUnit.HALF_HOUR_OF_DAY); + dateFieldUnits.put("half-hour", DataCubeDateTimeUnit.HALF_HOUR_OF_DAY); + dateFieldUnits.put("15m", DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY); + dateFieldUnits.put("quarter-hour", DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY); DATE_FIELD_UNITS = unmodifiableMap(dateFieldUnits); } private final String shortName; - ExtendedDateTimeUnit(String shortName) { + DataCubeDateTimeUnit(String shortName) { this.shortName = shortName; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java index 026e0259a5a84..438b8bb25bd3f 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java @@ -10,15 +10,16 @@ import org.opensearch.common.Rounding; import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DataCubeDateTimeUnit; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; -import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.test.OpenSearchTestCase; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import static org.opensearch.index.compositeindex.datacube.DateDimension.DateTimeUnitComparator.ORDERED_DATE_TIME_UNIT; @@ -26,7 +27,7 @@ public class DateDimensionTests extends OpenSearchTestCase { public void testDateDimension() { String field = "timestamp"; List intervals = Arrays.asList( - ExtendedDateTimeUnit.HALF_HOUR_OF_DAY, + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY, new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR), new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY) ); @@ -50,9 +51,10 @@ public void testSetDimensionValuesWithMultipleIntervalsYear() { Long[] dims = new Long[4]; Long testValue = 1609459200000L; // 2021-01-01 00:00:00 UTC - int nextIndex = dateDimension.setDimensionValues(testValue, dims, 0); + AtomicInteger dimIndex = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { dims[dimIndex.getAndIncrement()] = value; }); - assertEquals(4, nextIndex); + assertEquals(4, dimIndex.get()); assertEquals(1609459200000L, (long) dims[0]); // Hour rounded assertEquals(1609459200000L, (long) dims[1]); // Day rounded assertEquals(1609459200000L, (long) dims[2]); // Month rounded @@ -62,42 +64,51 @@ public void testSetDimensionValuesWithMultipleIntervalsYear() { public void testSetDimensionValuesForHalfAndQuarterHour() { List intervals = Arrays.asList( - ExtendedDateTimeUnit.HALF_HOUR_OF_DAY, - ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY, + DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY ); DateDimension dateDimension = new DateDimension("timestamp", intervals, DateFieldMapper.Resolution.MILLISECONDS); Long[] dims = new Long[2]; long testValue = 1724230620123L; // August 21, 2024 8:57:00.123 UTC - int nextIndex = dateDimension.setDimensionValues(testValue, dims, 0); + AtomicInteger dimIndex = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { dims[dimIndex.getAndIncrement()] = value; }); - assertEquals(2, nextIndex); + assertEquals(2, dimIndex.get()); assertEquals(1724229900000L, (long) dims[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:45:00 UTC assertEquals(1724229000000L, (long) dims[1]); // Half hour rounded - Wed, 21 Aug 2024 08:30:00 UTC assertEquals(2, dateDimension.getNumSubDimensions()); - dims = new Long[2]; + Long[] dims1 = new Long[2]; testValue = 1724229899234L; // Wed, 21 Aug 2024 08:44:59 GMT - dateDimension.setDimensionValues(testValue, dims, 0); - assertEquals(2, nextIndex); - assertEquals(1724229000000L, (long) dims[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:30:00 UTC - assertEquals(1724229000000L, (long) dims[1]); // Half hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + + AtomicInteger dimIndex1 = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { dims1[dimIndex1.getAndIncrement()] = value; }); + + assertEquals(1724229000000L, (long) dims1[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + assertEquals(1724229000000L, (long) dims1[1]); // Half hour rounded - Wed, 21 Aug 2024 08:30:00 UTC assertEquals(2, dateDimension.getNumSubDimensions()); - dims = new Long[2]; + Long[] dims2 = new Long[2]; testValue = 1724229000123L; // Wed, 21 Aug 2024 08:30:00 GMT - dateDimension.setDimensionValues(testValue, dims, 0); - assertEquals(2, nextIndex); - assertEquals(1724229000000L, (long) dims[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:30:00 UTC - assertEquals(1724229000000L, (long) dims[1]); // Half hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + + AtomicInteger dimIndex2 = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { dims2[dimIndex2.getAndIncrement()] = value; }); + + assertEquals(2, dimIndex2.get()); + assertEquals(1724229000000L, (long) dims2[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + assertEquals(1724229000000L, (long) dims2[1]); // Half hour rounded - Wed, 21 Aug 2024 08:30:00 UTC assertEquals(2, dateDimension.getNumSubDimensions()); - dims = new Long[2]; + Long[] dims3 = new Long[2]; testValue = 1724228940000L; // Wed, 21 Aug 2024 08:29:00 GMT - dateDimension.setDimensionValues(testValue, dims, 0); - assertEquals(2, nextIndex); - assertEquals(1724228100000L, (long) dims[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:15:00 UTC - assertEquals(1724227200000L, (long) dims[1]); // Half hour rounded - Wed, 21 Aug 2024 08:00:00 UTC + + AtomicInteger dimIndex3 = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { dims3[dimIndex3.getAndIncrement()] = value; }); + + assertEquals(2, dimIndex3.get()); + assertEquals(1724228100000L, (long) dims3[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:15:00 UTC + assertEquals(1724227200000L, (long) dims3[1]); // Half hour rounded - Wed, 21 Aug 2024 08:00:00 UTC assertEquals(2, dateDimension.getNumSubDimensions()); } @@ -120,7 +131,9 @@ public void testRoundingAndSortingAllDateTimeUnitsNanos() { // Test rounding long testValueNanos = 1655293505382719622L; // 2022-06-15T11:45:05.382719622Z Long[] dims = new Long[allUnits.size()]; - dateDimension.setDimensionValues(testValueNanos, dims, 0); + + AtomicInteger dimIndex = new AtomicInteger(0); + dateDimension.setDimensionValues(testValueNanos, value -> { dims[dimIndex.getAndIncrement()] = value; }); // Expected rounded values (in nanoseconds) long secondRounded = 1655293505000L; // 2022-06-15T11:45:05Z @@ -169,7 +182,9 @@ public void testRoundingAndSortingAllDateTimeUnitsMillis() { // Test rounding long testValueNanos = 1724114825234L; // 2024-08-20T00:47:05.234Z Long[] dims = new Long[allUnits.size()]; - dateDimension.setDimensionValues(testValueNanos, dims, 0); + + AtomicInteger dimIndex = new AtomicInteger(0); + dateDimension.setDimensionValues(testValueNanos, value -> { dims[dimIndex.getAndIncrement()] = value; }); // Expected rounded values (in millis) long secondRounded = 1724114825000L; // 2024-08-20T00:47:05.000Z @@ -209,8 +224,8 @@ private static List getAllTimeUnits() { new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR), new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY), new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH), - ExtendedDateTimeUnit.HALF_HOUR_OF_DAY, - ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY, + DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY ); } @@ -268,7 +283,7 @@ private static void assertTimeUnits( } } - public void testGetDimensionFieldsNames() { + public void testGetSubDimensionNames() { DateDimension dateDimension = new DateDimension( "timestamp", Arrays.asList( @@ -278,7 +293,7 @@ public void testGetDimensionFieldsNames() { DateFieldMapper.Resolution.MILLISECONDS ); - List fields = dateDimension.getDimensionFieldsNames(); + List fields = dateDimension.getSubDimensionNames(); assertEquals(2, fields.size()); assertEquals("timestamp_hour", fields.get(0)); @@ -288,11 +303,11 @@ public void testGetDimensionFieldsNames() { public void testGetExtendedTimeUnitFieldsNames() { DateDimension dateDimension = new DateDimension( "timestamp", - Arrays.asList(ExtendedDateTimeUnit.HALF_HOUR_OF_DAY, ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY), + Arrays.asList(DataCubeDateTimeUnit.HALF_HOUR_OF_DAY, DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY), DateFieldMapper.Resolution.MILLISECONDS ); - List fields = dateDimension.getDimensionFieldsNames(); + List fields = dateDimension.getSubDimensionNames(); assertEquals(2, fields.size()); assertEquals("timestamp_quarter-hour", fields.get(0)); @@ -311,9 +326,10 @@ public void testSetDimensionValues() { Long[] dims = new Long[2]; Long testValue = 1609459200000L; // 2021-01-01 00:00:00 UTC - int nextIndex = dateDimension.setDimensionValues(testValue, dims, 0); + AtomicInteger dimIndex = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { dims[dimIndex.getAndIncrement()] = value; }); - assertEquals(2, nextIndex); + assertEquals(2, dimIndex.get()); assertEquals(1609459200000L, (long) dims[0]); // Hour rounded assertEquals(1609459200000L, (long) dims[1]); // Year rounded } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index ccdd779fc4a37..dc8b3320f3de2 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -59,7 +59,7 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( int dimIndex = 0; for (int i = 0; i < dimensionsSplitOrder.size(); i++) { Dimension dimension = dimensionsSplitOrder.get(i); - for (String name : dimension.getDimensionFieldsNames()) { + for (String name : dimension.getSubDimensionNames()) { dimensionReaders[dimIndex] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(name)); dimIndex++; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 329487961315f..e9fb802c8d3ae 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -55,9 +55,9 @@ import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DataCubeDateTimeUnit; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; -import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import org.opensearch.index.mapper.ContentPath; @@ -1871,7 +1871,7 @@ private List getStarTreeDimensionNames(List dimensionsOrder) List dimensionNames = new ArrayList<>(); for (Dimension dimension : dimensionsOrder) { - dimensionNames.addAll(dimension.getDimensionFieldsNames()); + dimensionNames.addAll(dimension.getSubDimensionNames()); } return dimensionNames; @@ -4264,9 +4264,15 @@ public void testFlushFlowWithTimestamps() throws IOException { compositeField = getStarTreeFieldWithDateDimension(); SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); - SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList2, docsWithField2)); - SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(metricsList, metricsWithField)); - SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(metricsList, metricsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, @@ -4474,7 +4480,7 @@ private StarTreeField getStarTreeFieldWithDateDimension() { List intervals = new ArrayList<>(); intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR)); intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); - intervals.add(ExtendedDateTimeUnit.HALF_HOUR_OF_DAY); + intervals.add(DataCubeDateTimeUnit.HALF_HOUR_OF_DAY); Dimension d1 = new DateDimension("field1", intervals, DateFieldMapper.Resolution.MILLISECONDS); Dimension d2 = new NumericDimension("field3"); Metric m1 = new Metric("field2", List.of(MetricStat.VALUE_COUNT, MetricStat.SUM)); diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index dd3fbde1bea60..1869f0248502a 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -27,9 +27,9 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DataCubeDateTimeUnit; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; -import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit; import org.junit.After; import org.junit.Before; @@ -230,10 +230,10 @@ public void testValidStarTreeDefaults() throws IOException { assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); List expectedDimensionFields = Arrays.asList("@timestamp_minute", "@timestamp_half-hour"); - assertEquals(expectedDimensionFields, dateDim.getDimensionFieldsNames()); + assertEquals(expectedDimensionFields, dateDim.getSubDimensionNames()); List expectedTimeUnits = Arrays.asList( new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR), - ExtendedDateTimeUnit.HALF_HOUR_OF_DAY + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY ); for (int i = 0; i < expectedTimeUnits.size(); i++) { assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getIntervals().get(i).shortName()); @@ -265,9 +265,9 @@ public void testValidStarTreeDateDims() throws IOException { assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); List expectedDimensionFields = Arrays.asList("@timestamp_half-hour", "@timestamp_week", "@timestamp_month"); - assertEquals(expectedDimensionFields, dateDim.getDimensionFieldsNames()); + assertEquals(expectedDimensionFields, dateDim.getSubDimensionNames()); List expectedTimeUnits = Arrays.asList( - ExtendedDateTimeUnit.HALF_HOUR_OF_DAY, + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY, new DateTimeUnitAdapter(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR), new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR) );