Skip to content

Commit

Permalink
fixing build
Browse files Browse the repository at this point in the history
Signed-off-by: Bharathwaj G <bharath78910@gmail.com>
  • Loading branch information
bharath-techie committed Oct 17, 2024
1 parent a8652ff commit a6a2418
Show file tree
Hide file tree
Showing 4 changed files with 124 additions and 45 deletions.
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641))
- Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993))
- Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383))

- [Star tree] Adding date field rounding support in star tree ([#15249](https://github.com/opensearch-project/OpenSearch/issues/15249))
- [Star tree] Support for keyword changes as part of star tree mapper ([#16103](https://github.com/opensearch-project/OpenSearch/issues/16103))
### Dependencies
- Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578))
- Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.IndexOutput;
import org.opensearch.common.annotation.ExperimentalApi;
Expand All @@ -35,7 +34,6 @@
import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues;
import org.opensearch.index.mapper.CompositeMappedFieldType;
import org.opensearch.index.mapper.DocCountFieldMapper;
import org.opensearch.index.mapper.KeywordFieldMapper;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.index.mapper.StarTreeMapper;

Expand Down Expand Up @@ -84,7 +82,14 @@ public Composite912DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState
this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes();
compositeFieldSet = new HashSet<>();
segmentFieldSet = new HashSet<>();
addStarTreeSupportedFieldsFromSegment();
// TODO : add integ test for this
for (FieldInfo fi : this.state.fieldInfos) {
if (DocValuesType.SORTED_NUMERIC.equals(fi.getDocValuesType())) {
segmentFieldSet.add(fi.name);
} else if (fi.name.equals(DocCountFieldMapper.NAME)) {
segmentFieldSet.add(fi.name);
}
}
for (CompositeMappedFieldType type : compositeMappedFieldTypes) {
compositeFieldSet.addAll(type.fields());
}
Expand Down Expand Up @@ -143,19 +148,6 @@ public Composite912DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState
segmentHasCompositeFields = Collections.disjoint(segmentFieldSet, compositeFieldSet) == false;
}

private void addStarTreeSupportedFieldsFromSegment() {
// TODO : add integ test for this
for (FieldInfo fi : this.state.fieldInfos) {
if (DocValuesType.SORTED_NUMERIC.equals(fi.getDocValuesType())) {
segmentFieldSet.add(fi.name);
} else if (DocValuesType.SORTED_SET.equals(fi.getDocValuesType())) {
segmentFieldSet.add(fi.name);
} else if (fi.name.equals(DocCountFieldMapper.NAME)) {
segmentFieldSet.add(fi.name);
}
}
}

@Override
public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
delegate.addNumericField(field, valuesProducer);
Expand Down Expand Up @@ -187,10 +179,6 @@ public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProdu
@Override
public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
delegate.addSortedSetField(field, valuesProducer);
// Perform this only during flush flow
if (mergeState.get() == null && segmentHasCompositeFields) {
createCompositeIndicesIfPossible(valuesProducer, field);
}
}

@Override
Expand Down Expand Up @@ -247,7 +235,6 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer,
* Add empty doc values for fields not present in segment
*/
private void addDocValuesForEmptyField(String compositeField) {
// special case for doc count
if (compositeField.equals(DocCountFieldMapper.NAME)) {
fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() {
@Override
Expand All @@ -256,29 +243,16 @@ public NumericDocValues getNumeric(FieldInfo field) {
}
});
} else {
if (isSortedSetField(compositeField)) {
fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() {
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) {
return DocValues.emptySortedSet();
}
});
} else {
fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() {
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) {
return DocValues.emptySortedNumeric();
}
});
}
fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() {
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) {
return DocValues.emptySortedNumeric();
}
});
}
compositeFieldSet.remove(compositeField);
}

private boolean isSortedSetField(String field) {
return mapperService.fieldType(field) instanceof KeywordFieldMapper.KeywordFieldType;
}

@Override
public void merge(MergeState mergeState) throws IOException {
this.mergeState.compareAndSet(null, mergeState);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.function.Consumer;

/**
* Composite index keyword dimension class
Expand All @@ -41,13 +42,12 @@ public int getNumSubDimensions() {
}

@Override
public int setDimensionValues(Long value, Long[] dims, int index) {
dims[index++] = value;
return index;
public void setDimensionValues(Long value, Consumer<Long> dimSetter) {
dimSetter.accept(value);
}

@Override
public List<String> getDimensionFieldsNames() {
public List<String> getSubDimensionNames() {
return List.of(field);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.index.compositeindex.datacube;

import org.opensearch.common.settings.Settings;
import org.opensearch.index.mapper.Mapper;
import org.opensearch.test.OpenSearchTestCase;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class DimensionFactoryTests extends OpenSearchTestCase {

public void testParseAndCreateDateDimension() {
String name = "dateDimension";
Map<String, Object> dimensionMap = new HashMap<>();
List<String> calendarIntervals = Arrays.asList("day", "month");
dimensionMap.put("calendar_intervals", calendarIntervals);

Mapper.TypeParser.ParserContext mockContext = mock(Mapper.TypeParser.ParserContext.class);
when(mockContext.getSettings()).thenReturn(Settings.EMPTY);

Dimension dimension = DimensionFactory.parseAndCreateDimension(name, DateDimension.DATE, dimensionMap, mockContext);

assertTrue(dimension instanceof DateDimension);
assertEquals(2, dimension.getNumSubDimensions());
for (String interval : calendarIntervals) {
assertTrue(dimension.getSubDimensionNames().contains(name + "_" + interval));
}
assertEquals(name, dimension.getField());
DateDimension dateDimension = (DateDimension) dimension;
assertEquals(2, dateDimension.getIntervals().size());
}

public void testParseAndCreateNumericDimension() {
String name = "numericDimension";
Dimension dimension = DimensionFactory.parseAndCreateDimension(name, NumericDimension.NUMERIC, new HashMap<>(), null);

assertTrue(dimension instanceof NumericDimension);
assertEquals(1, dimension.getNumSubDimensions());
assertTrue(dimension.getSubDimensionNames().contains(name));
assertEquals(name, dimension.getField());
}

public void testParseAndCreateKeywordDimension() {
String name = "keywordDimension";
Dimension dimension = DimensionFactory.parseAndCreateDimension(name, KeywordDimension.KEYWORD, new HashMap<>(), null);
KeywordDimension kd = new KeywordDimension(name);
assertTrue(dimension instanceof KeywordDimension);
assertEquals(1, dimension.getNumSubDimensions());
assertTrue(dimension.getSubDimensionNames().contains(name));
assertEquals(dimension, kd);
assertEquals(name, dimension.getField());
List<Long> dimValue = new ArrayList<>();
kd.setDimensionValues(1L, dimValue::add);
assertEquals((Long) 1L, dimValue.get(0));
}

public void testParseAndCreateDimensionWithUnsupportedType() {
String name = "unsupportedDimension";
String unsupportedType = "unsupported";

IllegalArgumentException exception = expectThrows(
IllegalArgumentException.class,
() -> DimensionFactory.parseAndCreateDimension(name, unsupportedType, new HashMap<>(), null)
);
assertTrue(exception.getMessage().contains("unsupported field type"));
}

public void testParseAndCreateDimensionWithBuilder() {
String name = "builderDimension";
Mapper.Builder mockBuilder = mock(Mapper.Builder.class);
when(mockBuilder.getSupportedDataCubeDimensionType()).thenReturn(java.util.Optional.of(DimensionType.KEYWORD));

Dimension dimension = DimensionFactory.parseAndCreateDimension(name, mockBuilder, new HashMap<>(), null);

assertTrue(dimension instanceof KeywordDimension);
assertEquals(name, dimension.getField());
}

public void testParseAndCreateDimensionWithUnsupportedBuilder() {
String name = "unsupportedBuilderDimension";
Mapper.Builder mockBuilder = mock(Mapper.Builder.class);
when(mockBuilder.getSupportedDataCubeDimensionType()).thenReturn(java.util.Optional.empty());

IllegalArgumentException exception = expectThrows(
IllegalArgumentException.class,
() -> DimensionFactory.parseAndCreateDimension(name, mockBuilder, new HashMap<>(), null)
);
assertTrue(exception.getMessage().contains("unsupported field type"));
}
}

0 comments on commit a6a2418

Please sign in to comment.