From dfdbdfc37a6c3004d8f43016486174671dda9fe9 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Tue, 8 Oct 2024 18:19:20 +0530 Subject: [PATCH] fixing build Signed-off-by: Bharathwaj G --- .../Composite912DocValuesWriter.java | 54 +++----- .../datacube/KeywordDimension.java | 8 +- .../datacube/DimensionFactoryTests.java | 116 ++++++++++++++++++ 3 files changed, 134 insertions(+), 44 deletions(-) create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/DimensionFactoryTests.java diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java index 2c4f5dc4dd464..2225870afae8e 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java @@ -21,7 +21,6 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.ExperimentalApi; @@ -35,7 +34,6 @@ import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.mapper.CompositeMappedFieldType; import org.opensearch.index.mapper.DocCountFieldMapper; -import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.StarTreeMapper; @@ -84,7 +82,14 @@ public Composite912DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes(); compositeFieldSet = new HashSet<>(); segmentFieldSet = new HashSet<>(); - addStarTreeSupportedFieldsFromSegment(); + // TODO : add integ test for this + for (FieldInfo fi : this.state.fieldInfos) { + if (DocValuesType.SORTED_NUMERIC.equals(fi.getDocValuesType())) { + segmentFieldSet.add(fi.name); + } else if (fi.name.equals(DocCountFieldMapper.NAME)) { + segmentFieldSet.add(fi.name); + } + } for (CompositeMappedFieldType type : compositeMappedFieldTypes) { compositeFieldSet.addAll(type.fields()); } @@ -143,19 +148,6 @@ public Composite912DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentHasCompositeFields = Collections.disjoint(segmentFieldSet, compositeFieldSet) == false; } - private void addStarTreeSupportedFieldsFromSegment() { - // TODO : add integ test for this - for (FieldInfo fi : this.state.fieldInfos) { - if (DocValuesType.SORTED_NUMERIC.equals(fi.getDocValuesType())) { - segmentFieldSet.add(fi.name); - } else if (DocValuesType.SORTED_SET.equals(fi.getDocValuesType())) { - segmentFieldSet.add(fi.name); - } else if (fi.name.equals(DocCountFieldMapper.NAME)) { - segmentFieldSet.add(fi.name); - } - } - } - @Override public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { delegate.addNumericField(field, valuesProducer); @@ -187,10 +179,6 @@ public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProdu @Override public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { delegate.addSortedSetField(field, valuesProducer); - // Perform this only during flush flow - if (mergeState.get() == null && segmentHasCompositeFields) { - createCompositeIndicesIfPossible(valuesProducer, field); - } } @Override @@ -247,7 +235,6 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, * Add empty doc values for fields not present in segment */ private void addDocValuesForEmptyField(String compositeField) { - // special case for doc count if (compositeField.equals(DocCountFieldMapper.NAME)) { fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() { @Override @@ -256,29 +243,16 @@ public NumericDocValues getNumeric(FieldInfo field) { } }); } else { - if (isSortedSetField(compositeField)) { - fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() { - @Override - public SortedSetDocValues getSortedSet(FieldInfo field) { - return DocValues.emptySortedSet(); - } - }); - } else { - fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() { - @Override - public SortedNumericDocValues getSortedNumeric(FieldInfo field) { - return DocValues.emptySortedNumeric(); - } - }); - } + fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) { + return DocValues.emptySortedNumeric(); + } + }); } compositeFieldSet.remove(compositeField); } - private boolean isSortedSetField(String field) { - return mapperService.fieldType(field) instanceof KeywordFieldMapper.KeywordFieldType; - } - @Override public void merge(MergeState mergeState) throws IOException { this.mergeState.compareAndSet(null, mergeState); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/KeywordDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/KeywordDimension.java index dd6cedade43ff..ce9968a67adda 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/KeywordDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/KeywordDimension.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.util.List; import java.util.Objects; +import java.util.function.Consumer; /** * Composite index keyword dimension class @@ -41,13 +42,12 @@ public int getNumSubDimensions() { } @Override - public int setDimensionValues(Long value, Long[] dims, int index) { - dims[index++] = value; - return index; + public void setDimensionValues(Long value, Consumer dimSetter) { + dimSetter.accept(value); } @Override - public List getDimensionFieldsNames() { + public List getSubDimensionNames() { return List.of(field); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/DimensionFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/DimensionFactoryTests.java new file mode 100644 index 0000000000000..cb77ab419db3b --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/DimensionFactoryTests.java @@ -0,0 +1,116 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +import org.opensearch.common.settings.Settings; +import org.opensearch.index.mapper.Mapper; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DimensionFactoryTests extends OpenSearchTestCase { + + public void testParseAndCreateDateDimension() { + String name = "dateDimension"; + Map dimensionMap = new HashMap<>(); + List calendarIntervals = Arrays.asList("day", "month"); + dimensionMap.put("calendar_intervals", calendarIntervals); + + Mapper.TypeParser.ParserContext mockContext = mock(Mapper.TypeParser.ParserContext.class); + when(mockContext.getSettings()).thenReturn(Settings.EMPTY); + + Dimension dimension = DimensionFactory.parseAndCreateDimension(name, DateDimension.DATE, dimensionMap, mockContext); + + assertTrue(dimension instanceof DateDimension); + assertEquals(2, dimension.getNumSubDimensions()); + for (String interval : calendarIntervals) { + assertTrue(dimension.getSubDimensionNames().contains(name + "_" + interval)); + } + assertEquals(name, dimension.getField()); + DateDimension dateDimension = (DateDimension) dimension; + assertEquals(2, dateDimension.getIntervals().size()); + } + + public void testParseAndCreateNumericDimension() { + String name = "numericDimension"; + Dimension dimension = DimensionFactory.parseAndCreateDimension(name, NumericDimension.NUMERIC, new HashMap<>(), null); + + assertTrue(dimension instanceof NumericDimension); + assertEquals(1, dimension.getNumSubDimensions()); + assertTrue(dimension.getSubDimensionNames().contains(name)); + assertEquals(name, dimension.getField()); + } + + public void testParseAndCreateKeywordDimension() { + String name = "keywordDimension"; + Dimension dimension = DimensionFactory.parseAndCreateDimension(name, KeywordDimension.KEYWORD, new HashMap<>(), null); + KeywordDimension kd = new KeywordDimension(name); + assertTrue(dimension instanceof KeywordDimension); + assertEquals(1, dimension.getNumSubDimensions()); + assertTrue(dimension.getSubDimensionNames().contains(name)); + assertEquals(dimension, kd); + assertEquals(name, dimension.getField()); + List dimValue = new ArrayList<>(); + kd.setDimensionValues(1L, dimValue::add); + assertEquals((Long) 1L, dimValue.get(0)); + } + + public void testParseAndCreateDimensionWithUnsupportedType() { + String name = "unsupportedDimension"; + String unsupportedType = "unsupported"; + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> DimensionFactory.parseAndCreateDimension(name, unsupportedType, new HashMap<>(), null) + ); + assertTrue(exception.getMessage().contains("unsupported field type")); + } + + public void testParseAndCreateDimensionWithBuilder() { + String name = "builderDimension"; + Mapper.Builder mockBuilder = mock(Mapper.Builder.class); + when(mockBuilder.getSupportedDataCubeDimensionType()).thenReturn(java.util.Optional.of(DimensionType.KEYWORD)); + + Dimension dimension = DimensionFactory.parseAndCreateDimension(name, mockBuilder, new HashMap<>(), null); + + assertTrue(dimension instanceof KeywordDimension); + assertEquals(name, dimension.getField()); + } + + public void testParseAndCreateDimensionWithEmptyBuilder() { + String name = "unsupportedBuilderDimension"; + Mapper.Builder mockBuilder = mock(Mapper.Builder.class); + when(mockBuilder.getSupportedDataCubeDimensionType()).thenReturn(java.util.Optional.empty()); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> DimensionFactory.parseAndCreateDimension(name, mockBuilder, new HashMap<>(), null) + ); + assertTrue(exception.getMessage().contains("unsupported field type")); + } + + public void testParseAndCreateDimensionWithUnsupportedBuilder() { + String name = "unsupportedBuilderDimension"; + Mapper.Builder mockBuilder = mock(Mapper.Builder.class); + when(mockBuilder.getSupportedDataCubeDimensionType()).thenReturn(java.util.Optional.of(DimensionType.KEYWORD)); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> DimensionFactory.parseAndCreateDimension(name, mockBuilder, new HashMap<>(), null) + ); + assertTrue(exception.getMessage().contains("unsupported field type")); + } +}