diff --git a/CHANGELOG.md b/CHANGELOG.md index e52c15c77..4f9e75b7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix typo for sparse encoding processor factory([#578](https://github.com/opensearch-project/neural-search/pull/578)) - Add non-null check for queryBuilder in NeuralQueryEnricherProcessor ([#615](https://github.com/opensearch-project/neural-search/pull/615)) ### Infrastructure -- Adding integ tests for aggregations when they're bundled with hybrid query ([#632](https://github.com/opensearch-project/neural-search/pull/632)) +- Adding integration tests for scenario of hybrid query with aggregations ([#632](https://github.com/opensearch-project/neural-search/pull/632)) ### Documentation ### Maintenance ### Refactoring diff --git a/src/test/java/org/opensearch/neuralsearch/query/aggregation/BaseAggregationsWithHybridQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/aggregation/BaseAggregationsWithHybridQueryIT.java index 44d8438af..83bec7ce4 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/aggregation/BaseAggregationsWithHybridQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/aggregation/BaseAggregationsWithHybridQueryIT.java @@ -5,7 +5,7 @@ package org.opensearch.neuralsearch.query.aggregation; import lombok.SneakyThrows; -import org.junit.Before; +import org.junit.BeforeClass; import org.opensearch.neuralsearch.BaseNeuralSearchIT; import java.util.ArrayList; @@ -31,40 +31,40 @@ public class BaseAggregationsWithHybridQueryIT extends BaseNeuralSearchIT { protected static final String TEST_QUERY_TEXT4 = "cost"; protected static final String TEST_QUERY_TEXT5 = "welcome"; protected static final String TEST_NESTED_TYPE_FIELD_NAME_1 = "user"; - protected static final String NESTED_FIELD_1 = "firstname"; - protected static final String NESTED_FIELD_2 = "lastname"; - protected static final String NESTED_FIELD_1_VALUE_1 = "john"; - protected static final String NESTED_FIELD_2_VALUE_1 = "black"; - protected static final String NESTED_FIELD_1_VALUE_2 = "frodo"; - protected static final String NESTED_FIELD_2_VALUE_2 = "baggins"; - protected static final String NESTED_FIELD_1_VALUE_3 = "mohammed"; - protected static final String NESTED_FIELD_2_VALUE_3 = "ezab"; - protected static final String NESTED_FIELD_1_VALUE_4 = "sun"; - protected static final String NESTED_FIELD_2_VALUE_4 = "wukong"; - protected static final String NESTED_FIELD_1_VALUE_5 = "vasilisa"; - protected static final String NESTED_FIELD_2_VALUE_5 = "the wise"; - protected static final String INTEGER_FIELD_1 = "doc_index"; - protected static final int INTEGER_FIELD_1_VALUE = 1234; - protected static final int INTEGER_FIELD_2_VALUE = 2345; - protected static final int INTEGER_FIELD_3_VALUE = 3456; - protected static final int INTEGER_FIELD_4_VALUE = 4567; - protected static final String KEYWORD_FIELD_1 = "doc_keyword"; - protected static final String KEYWORD_FIELD_1_VALUE = "workable"; - protected static final String KEYWORD_FIELD_2_VALUE = "angry"; - protected static final String KEYWORD_FIELD_3_VALUE = "likeable"; - protected static final String KEYWORD_FIELD_4_VALUE = "entire"; - protected static final String DATE_FIELD_1 = "doc_date"; - protected static final String DATE_FIELD_1_VALUE = "01/03/1995"; - private static final String DATE_FIELD_2_VALUE = "05/02/2015"; - protected static final String DATE_FIELD_3_VALUE = "07/23/2007"; - protected static final String DATE_FIELD_4_VALUE = "08/21/2012"; + protected static final String NESTED_FIELD_FIRSTNAME = "firstname"; + protected static final String NESTED_FIELD_LASTNAME = "lastname"; + protected static final String NESTED_FIELD_FIRSTNAME_JOHN = "john"; + protected static final String NESTED_FIELD_LASTNAME_BLACK = "black"; + protected static final String NESTED_FIELD_FIRSTNAME_FRODO = "frodo"; + protected static final String NESTED_FIELD_LASTNAME_BAGGINS = "baggins"; + protected static final String NESTED_FIELD_FIRSTNAME_MOHAMMED = "mohammed"; + protected static final String NESTED_FIELD_LASTNAME_EZAB = "ezab"; + protected static final String NESTED_FIELD_FIRSTNAME_SUN = "sun"; + protected static final String NESTED_FIELD_LASTNAME_WUKONG = "wukong"; + protected static final String NESTED_FIELD_FIRSTNAME_VASILISA = "vasilisa"; + protected static final String NESTED_FIELD_LASTNAME_WISE = "the wise"; + protected static final String INTEGER_FIELD_DOCINDEX = "doc_index"; + protected static final int INTEGER_FIELD_DOCINDEX_1234 = 1234; + protected static final int INTEGER_FIELD_DOCINDEX_2345 = 2345; + protected static final int INTEGER_FIELD_DOCINDEX_3456 = 3456; + protected static final int INTEGER_FIELD_DOCINDEX_4567 = 4567; + protected static final String KEYWORD_FIELD_DOCKEYWORD = "doc_keyword"; + protected static final String KEYWORD_FIELD_DOCKEYWORD_WORKABLE = "workable"; + protected static final String KEYWORD_FIELD_DOCKEYWORD_ANGRY = "angry"; + protected static final String KEYWORD_FIELD_DOCKEYWORD_LIKABLE = "likeable"; + protected static final String KEYWORD_FIELD_DOCKEYWORD_ENTIRE = "entire"; + protected static final String DATE_FIELD = "doc_date"; + protected static final String DATE_FIELD_01031995 = "01/03/1995"; + protected static final String DATE_FIELD_05022015 = "05/02/2015"; + protected static final String DATE_FIELD_07232007 = "07/23/2007"; + protected static final String DATE_FIELD_08212012 = "08/21/2012"; protected static final String INTEGER_FIELD_PRICE = "doc_price"; - protected static final int INTEGER_FIELD_PRICE_1_VALUE = 130; - protected static final int INTEGER_FIELD_PRICE_2_VALUE = 100; - protected static final int INTEGER_FIELD_PRICE_3_VALUE = 200; - protected static final int INTEGER_FIELD_PRICE_4_VALUE = 25; - protected static final int INTEGER_FIELD_PRICE_5_VALUE = 30; - protected static final int INTEGER_FIELD_PRICE_6_VALUE = 350; + protected static final int INTEGER_FIELD_PRICE_130 = 130; + protected static final int INTEGER_FIELD_PRICE_100 = 100; + protected static final int INTEGER_FIELD_PRICE_200 = 200; + protected static final int INTEGER_FIELD_PRICE_25 = 25; + protected static final int INTEGER_FIELD_PRICE_30 = 30; + protected static final int INTEGER_FIELD_PRICE_350 = 350; protected static final String BUCKET_AGG_DOC_COUNT_FIELD = "doc_count"; protected static final String BUCKETS_AGGREGATION_NAME_1 = "date_buckets_1"; protected static final String BUCKETS_AGGREGATION_NAME_2 = "date_buckets_2"; @@ -79,10 +79,14 @@ public class BaseAggregationsWithHybridQueryIT extends BaseNeuralSearchIT { protected static final String DATE_AGGREGATION_NAME = "date_aggregation"; protected static final String CLUSTER_SETTING_CONCURRENT_SEGMENT_SEARCH = "search.concurrent_segment_search.enabled"; - @Before - public void setUp() throws Exception { - super.setUp(); - updateClusterSettings(); + @BeforeClass + @SneakyThrows + public static void setUpCluster() { + // we need new instance because we're calling non-static methods from static method. + // main purpose is to minimize network calls, initialization is only needed once + BaseAggregationsWithHybridQueryIT instance = new BaseAggregationsWithHybridQueryIT(); + instance.initClient(); + instance.updateClusterSettings(); } @Override @@ -107,10 +111,10 @@ protected void initializeIndexIfNotExist(String indexName) { indexName, buildIndexConfiguration( List.of(), - List.of(TEST_NESTED_TYPE_FIELD_NAME_1, NESTED_FIELD_1, NESTED_FIELD_2), - List.of(INTEGER_FIELD_1), - List.of(KEYWORD_FIELD_1), - List.of(DATE_FIELD_1), + List.of(TEST_NESTED_TYPE_FIELD_NAME_1, NESTED_FIELD_FIRSTNAME, NESTED_FIELD_LASTNAME), + List.of(INTEGER_FIELD_DOCINDEX), + List.of(KEYWORD_FIELD_DOCKEYWORD), + List.of(DATE_FIELD), 3 ), "" @@ -124,13 +128,13 @@ protected void initializeIndexIfNotExist(String indexName) { Collections.singletonList(TEST_TEXT_FIELD_NAME_1), Collections.singletonList(TEST_DOC_TEXT1), List.of(TEST_NESTED_TYPE_FIELD_NAME_1), - List.of(Map.of(NESTED_FIELD_1, NESTED_FIELD_1_VALUE_1, NESTED_FIELD_2, NESTED_FIELD_2_VALUE_1)), - List.of(INTEGER_FIELD_1, INTEGER_FIELD_PRICE), - List.of(INTEGER_FIELD_1_VALUE, INTEGER_FIELD_PRICE_1_VALUE), - List.of(KEYWORD_FIELD_1), - List.of(KEYWORD_FIELD_1_VALUE), - List.of(DATE_FIELD_1), - List.of(DATE_FIELD_1_VALUE) + List.of(Map.of(NESTED_FIELD_FIRSTNAME, NESTED_FIELD_FIRSTNAME_JOHN, NESTED_FIELD_LASTNAME, NESTED_FIELD_LASTNAME_BLACK)), + List.of(INTEGER_FIELD_DOCINDEX, INTEGER_FIELD_PRICE), + List.of(INTEGER_FIELD_DOCINDEX_1234, INTEGER_FIELD_PRICE_130), + List.of(KEYWORD_FIELD_DOCKEYWORD), + List.of(KEYWORD_FIELD_DOCKEYWORD_WORKABLE), + List.of(DATE_FIELD), + List.of(DATE_FIELD_01031995) ); addKnnDoc( indexName, @@ -140,13 +144,13 @@ protected void initializeIndexIfNotExist(String indexName) { Collections.singletonList(TEST_TEXT_FIELD_NAME_1), Collections.singletonList(TEST_DOC_TEXT3), List.of(TEST_NESTED_TYPE_FIELD_NAME_1), - List.of(Map.of(NESTED_FIELD_1, NESTED_FIELD_1_VALUE_2, NESTED_FIELD_2, NESTED_FIELD_2_VALUE_2)), - List.of(INTEGER_FIELD_1, INTEGER_FIELD_PRICE), - List.of(INTEGER_FIELD_2_VALUE, INTEGER_FIELD_PRICE_2_VALUE), + List.of(Map.of(NESTED_FIELD_FIRSTNAME, NESTED_FIELD_FIRSTNAME_FRODO, NESTED_FIELD_LASTNAME, NESTED_FIELD_LASTNAME_BAGGINS)), + List.of(INTEGER_FIELD_DOCINDEX, INTEGER_FIELD_PRICE), + List.of(INTEGER_FIELD_DOCINDEX_2345, INTEGER_FIELD_PRICE_100), List.of(), List.of(), - List.of(DATE_FIELD_1), - List.of(DATE_FIELD_2_VALUE) + List.of(DATE_FIELD), + List.of(DATE_FIELD_05022015) ); addKnnDoc( indexName, @@ -156,13 +160,13 @@ protected void initializeIndexIfNotExist(String indexName) { Collections.singletonList(TEST_TEXT_FIELD_NAME_1), Collections.singletonList(TEST_DOC_TEXT2), List.of(TEST_NESTED_TYPE_FIELD_NAME_1), - List.of(Map.of(NESTED_FIELD_1, NESTED_FIELD_1_VALUE_3, NESTED_FIELD_2, NESTED_FIELD_2_VALUE_3)), + List.of(Map.of(NESTED_FIELD_FIRSTNAME, NESTED_FIELD_FIRSTNAME_MOHAMMED, NESTED_FIELD_LASTNAME, NESTED_FIELD_LASTNAME_EZAB)), List.of(INTEGER_FIELD_PRICE), - List.of(INTEGER_FIELD_PRICE_3_VALUE), - List.of(KEYWORD_FIELD_1), - List.of(KEYWORD_FIELD_2_VALUE), - List.of(DATE_FIELD_1), - List.of(DATE_FIELD_3_VALUE) + List.of(INTEGER_FIELD_PRICE_200), + List.of(KEYWORD_FIELD_DOCKEYWORD), + List.of(KEYWORD_FIELD_DOCKEYWORD_ANGRY), + List.of(DATE_FIELD), + List.of(DATE_FIELD_07232007) ); addKnnDoc( indexName, @@ -172,13 +176,13 @@ protected void initializeIndexIfNotExist(String indexName) { Collections.singletonList(TEST_TEXT_FIELD_NAME_1), Collections.singletonList(TEST_DOC_TEXT4), List.of(TEST_NESTED_TYPE_FIELD_NAME_1), - List.of(Map.of(NESTED_FIELD_1, NESTED_FIELD_1_VALUE_4, NESTED_FIELD_2, NESTED_FIELD_2_VALUE_4)), - List.of(INTEGER_FIELD_1, INTEGER_FIELD_PRICE), - List.of(INTEGER_FIELD_3_VALUE, INTEGER_FIELD_PRICE_4_VALUE), - List.of(KEYWORD_FIELD_1), - List.of(KEYWORD_FIELD_3_VALUE), - List.of(DATE_FIELD_1), - List.of(DATE_FIELD_2_VALUE) + List.of(Map.of(NESTED_FIELD_FIRSTNAME, NESTED_FIELD_FIRSTNAME_SUN, NESTED_FIELD_LASTNAME, NESTED_FIELD_LASTNAME_WUKONG)), + List.of(INTEGER_FIELD_DOCINDEX, INTEGER_FIELD_PRICE), + List.of(INTEGER_FIELD_DOCINDEX_3456, INTEGER_FIELD_PRICE_25), + List.of(KEYWORD_FIELD_DOCKEYWORD), + List.of(KEYWORD_FIELD_DOCKEYWORD_LIKABLE), + List.of(DATE_FIELD), + List.of(DATE_FIELD_05022015) ); addKnnDoc( indexName, @@ -189,12 +193,12 @@ protected void initializeIndexIfNotExist(String indexName) { Collections.singletonList(TEST_DOC_TEXT5), List.of(), List.of(), - List.of(INTEGER_FIELD_1, INTEGER_FIELD_PRICE), - List.of(INTEGER_FIELD_3_VALUE, INTEGER_FIELD_PRICE_5_VALUE), - List.of(KEYWORD_FIELD_1), - List.of(KEYWORD_FIELD_4_VALUE), - List.of(DATE_FIELD_1), - List.of(DATE_FIELD_4_VALUE) + List.of(INTEGER_FIELD_DOCINDEX, INTEGER_FIELD_PRICE), + List.of(INTEGER_FIELD_DOCINDEX_3456, INTEGER_FIELD_PRICE_30), + List.of(KEYWORD_FIELD_DOCKEYWORD), + List.of(KEYWORD_FIELD_DOCKEYWORD_ENTIRE), + List.of(DATE_FIELD), + List.of(DATE_FIELD_08212012) ); addKnnDoc( indexName, @@ -204,13 +208,13 @@ protected void initializeIndexIfNotExist(String indexName) { Collections.singletonList(TEST_TEXT_FIELD_NAME_1), Collections.singletonList(TEST_DOC_TEXT6), List.of(TEST_NESTED_TYPE_FIELD_NAME_1), - List.of(Map.of(NESTED_FIELD_1, NESTED_FIELD_1_VALUE_5, NESTED_FIELD_2, NESTED_FIELD_2_VALUE_5)), - List.of(INTEGER_FIELD_1, INTEGER_FIELD_PRICE), - List.of(INTEGER_FIELD_4_VALUE, INTEGER_FIELD_PRICE_6_VALUE), - List.of(KEYWORD_FIELD_1), - List.of(KEYWORD_FIELD_4_VALUE), - List.of(DATE_FIELD_1), - List.of(DATE_FIELD_4_VALUE) + List.of(Map.of(NESTED_FIELD_FIRSTNAME, NESTED_FIELD_FIRSTNAME_VASILISA, NESTED_FIELD_LASTNAME, NESTED_FIELD_LASTNAME_WISE)), + List.of(INTEGER_FIELD_DOCINDEX, INTEGER_FIELD_PRICE), + List.of(INTEGER_FIELD_DOCINDEX_4567, INTEGER_FIELD_PRICE_350), + List.of(KEYWORD_FIELD_DOCKEYWORD), + List.of(KEYWORD_FIELD_DOCKEYWORD_ENTIRE), + List.of(DATE_FIELD), + List.of(DATE_FIELD_08212012) ); } } diff --git a/src/test/java/org/opensearch/neuralsearch/query/aggregation/BucketAggregationsWithHybridQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/aggregation/BucketAggregationsWithHybridQueryIT.java index 345d805a8..e0ce1cc87 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/aggregation/BucketAggregationsWithHybridQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/aggregation/BucketAggregationsWithHybridQueryIT.java @@ -236,8 +236,8 @@ private void testAvgNestedIntoFilter() throws IOException { AggregationBuilder aggsBuilder = AggregationBuilders.filter( GENERIC_AGGREGATION_NAME, - QueryBuilders.rangeQuery(INTEGER_FIELD_1).lte(3000) - ).subAggregation(AggregationBuilders.avg(AVG_AGGREGATION_NAME).field(INTEGER_FIELD_1)); + QueryBuilders.rangeQuery(INTEGER_FIELD_DOCINDEX).lte(3000) + ).subAggregation(AggregationBuilders.avg(AVG_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX)); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS @@ -259,9 +259,9 @@ private void testSumNestedIntoFilters() throws IOException { AggregationBuilder aggsBuilder = AggregationBuilders.filters( GENERIC_AGGREGATION_NAME, - QueryBuilders.rangeQuery(INTEGER_FIELD_1).lte(3000), - QueryBuilders.termQuery(KEYWORD_FIELD_1, KEYWORD_FIELD_1_VALUE) - ).otherBucket(true).subAggregation(AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_1)); + QueryBuilders.rangeQuery(INTEGER_FIELD_DOCINDEX).lte(3000), + QueryBuilders.termQuery(KEYWORD_FIELD_DOCKEYWORD, KEYWORD_FIELD_DOCKEYWORD_WORKABLE) + ).otherBucket(true).subAggregation(AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX)); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS @@ -305,7 +305,7 @@ private void testGlobalAggs() throws IOException { hybridQueryBuilderNeuralThenTerm.add(termQueryBuilder2); AggregationBuilder aggsBuilder = AggregationBuilders.global(GENERIC_AGGREGATION_NAME) - .subAggregation(AggregationBuilders.sum(AVG_AGGREGATION_NAME).field(INTEGER_FIELD_1)); + .subAggregation(AggregationBuilders.sum(AVG_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX)); Map searchResponseAsMap = executeQueryAndGetAggsResults( List.of(aggsBuilder), @@ -364,7 +364,7 @@ private void testNestedAggs() throws IOException { AggregationBuilder aggsBuilder = AggregationBuilders.nested(GENERIC_AGGREGATION_NAME, TEST_NESTED_TYPE_FIELD_NAME_1) .subAggregation( AggregationBuilders.terms(BUCKETS_AGGREGATION_NAME_1) - .field(String.join(".", TEST_NESTED_TYPE_FIELD_NAME_1, NESTED_FIELD_1)) + .field(String.join(".", TEST_NESTED_TYPE_FIELD_NAME_1, NESTED_FIELD_FIRSTNAME)) ); Map searchResponseAsMap = executeQueryAndGetAggsResults( @@ -387,17 +387,17 @@ private void testNestedAggs() throws IOException { Map firstBucket = buckets.get(0); assertEquals(2, firstBucket.size()); assertEquals(1, firstBucket.get(BUCKET_AGG_DOC_COUNT_FIELD)); - assertEquals(NESTED_FIELD_1_VALUE_2, firstBucket.get(KEY)); + assertEquals(NESTED_FIELD_FIRSTNAME_FRODO, firstBucket.get(KEY)); Map secondBucket = buckets.get(1); assertEquals(2, secondBucket.size()); assertEquals(1, secondBucket.get(BUCKET_AGG_DOC_COUNT_FIELD)); - assertEquals(NESTED_FIELD_1_VALUE_1, secondBucket.get(KEY)); + assertEquals(NESTED_FIELD_FIRSTNAME_JOHN, secondBucket.get(KEY)); Map thirdBucket = buckets.get(2); assertEquals(2, thirdBucket.size()); assertEquals(1, thirdBucket.get(BUCKET_AGG_DOC_COUNT_FIELD)); - assertEquals(NESTED_FIELD_1_VALUE_4, thirdBucket.get(KEY)); + assertEquals(NESTED_FIELD_FIRSTNAME_SUN, thirdBucket.get(KEY)); } finally { wipeOfTestResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, null, null, SEARCH_PIPELINE); } @@ -408,9 +408,9 @@ private void testDiversifiedSampler() throws IOException { prepareResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, SEARCH_PIPELINE); AggregationBuilder aggsBuilder = AggregationBuilders.diversifiedSampler(GENERIC_AGGREGATION_NAME) - .field(KEYWORD_FIELD_1) + .field(KEYWORD_FIELD_DOCKEYWORD) .shardSize(2) - .subAggregation(AggregationBuilders.terms(BUCKETS_AGGREGATION_NAME_1).field(KEYWORD_FIELD_1)); + .subAggregation(AggregationBuilders.terms(BUCKETS_AGGREGATION_NAME_1).field(KEYWORD_FIELD_DOCKEYWORD)); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, @@ -449,11 +449,11 @@ private void testAdjacencyMatrixAggs() throws IOException { GENERIC_AGGREGATION_NAME, Map.of( "grpA", - QueryBuilders.matchQuery(KEYWORD_FIELD_1, KEYWORD_FIELD_1_VALUE), + QueryBuilders.matchQuery(KEYWORD_FIELD_DOCKEYWORD, KEYWORD_FIELD_DOCKEYWORD_WORKABLE), "grpB", - QueryBuilders.matchQuery(KEYWORD_FIELD_1, KEYWORD_FIELD_2_VALUE), + QueryBuilders.matchQuery(KEYWORD_FIELD_DOCKEYWORD, KEYWORD_FIELD_DOCKEYWORD_ANGRY), "grpC", - QueryBuilders.matchQuery(KEYWORD_FIELD_1, KEYWORD_FIELD_3_VALUE) + QueryBuilders.matchQuery(KEYWORD_FIELD_DOCKEYWORD, KEYWORD_FIELD_DOCKEYWORD_LIKABLE) ) ); Map searchResponseAsMap = executeQueryAndGetAggsResults( @@ -483,8 +483,8 @@ private void testDateBucketedSumsPipelinedToBucketMinMaxSumAvgAggs() throws IOEx AggregationBuilder aggDateHisto = AggregationBuilders.dateHistogram(GENERIC_AGGREGATION_NAME) .calendarInterval(DateHistogramInterval.YEAR) - .field(DATE_FIELD_1) - .subAggregation(AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_1)); + .field(DATE_FIELD) + .subAggregation(AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX)); BucketMetricsPipelineAggregationBuilder aggAvgBucket = PipelineAggregatorBuilders .avgBucket(BUCKETS_AGGREGATION_NAME_1, GENERIC_AGGREGATION_NAME + ">" + SUM_AGGREGATION_NAME); @@ -558,8 +558,8 @@ private void testDateBucketedSumsPipelinedToBucketStatsAggs() throws IOException AggregationBuilder aggDateHisto = AggregationBuilders.dateHistogram(GENERIC_AGGREGATION_NAME) .calendarInterval(DateHistogramInterval.YEAR) - .field(DATE_FIELD_1) - .subAggregation(AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_1)); + .field(DATE_FIELD) + .subAggregation(AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX)); StatsBucketPipelineAggregationBuilder aggStatsBucket = PipelineAggregatorBuilders.statsBucket( BUCKETS_AGGREGATION_NAME_1, @@ -594,19 +594,21 @@ private void testDateBucketedSumsPipelinedToBucketScriptedAggs() throws IOExcept AggregationBuilder aggBuilder = AggregationBuilders.dateHistogram(DATE_AGGREGATION_NAME) .calendarInterval(DateHistogramInterval.YEAR) - .field(DATE_FIELD_1) + .field(DATE_FIELD) .subAggregations( - new AggregatorFactories.Builder().addAggregator(AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_1)) + new AggregatorFactories.Builder().addAggregator( + AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX) + ) .addAggregator( AggregationBuilders.filter( GENERIC_AGGREGATION_NAME, QueryBuilders.boolQuery() .should( QueryBuilders.boolQuery() - .should(QueryBuilders.termQuery(KEYWORD_FIELD_1, KEYWORD_FIELD_1_VALUE)) - .should(QueryBuilders.termQuery(KEYWORD_FIELD_1, KEYWORD_FIELD_2_VALUE)) + .should(QueryBuilders.termQuery(KEYWORD_FIELD_DOCKEYWORD, KEYWORD_FIELD_DOCKEYWORD_WORKABLE)) + .should(QueryBuilders.termQuery(KEYWORD_FIELD_DOCKEYWORD, KEYWORD_FIELD_DOCKEYWORD_ANGRY)) ) - .should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(KEYWORD_FIELD_1))) + .should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(KEYWORD_FIELD_DOCKEYWORD))) ).subAggregation(AggregationBuilders.sum(SUM_AGGREGATION_NAME_2).field(INTEGER_FIELD_PRICE)) ) .addPipelineAggregator( @@ -684,7 +686,7 @@ private void testSampler() throws IOException { AggregationBuilder aggsBuilder = AggregationBuilders.sampler(GENERIC_AGGREGATION_NAME) .shardSize(2) - .subAggregation(AggregationBuilders.terms(BUCKETS_AGGREGATION_NAME_1).field(KEYWORD_FIELD_1)); + .subAggregation(AggregationBuilders.terms(BUCKETS_AGGREGATION_NAME_1).field(KEYWORD_FIELD_DOCKEYWORD)); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, @@ -719,7 +721,7 @@ private void testTermsAggs() throws IOException { try { prepareResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, SEARCH_PIPELINE); - AggregationBuilder aggsBuilder = AggregationBuilders.terms(GENERIC_AGGREGATION_NAME).field(KEYWORD_FIELD_1); + AggregationBuilder aggsBuilder = AggregationBuilders.terms(GENERIC_AGGREGATION_NAME).field(KEYWORD_FIELD_DOCKEYWORD); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS @@ -734,10 +736,10 @@ private void testTermsAggs() throws IOException { assertEquals(2, buckets.size()); Map firstBucket = buckets.get(0); assertEquals(1, firstBucket.get(BUCKET_AGG_DOC_COUNT_FIELD)); - assertEquals(KEYWORD_FIELD_3_VALUE, firstBucket.get(KEY)); + assertEquals(KEYWORD_FIELD_DOCKEYWORD_LIKABLE, firstBucket.get(KEY)); Map secondBucket = buckets.get(1); assertEquals(1, secondBucket.get(BUCKET_AGG_DOC_COUNT_FIELD)); - assertEquals(KEYWORD_FIELD_1_VALUE, secondBucket.get(KEY)); + assertEquals(KEYWORD_FIELD_DOCKEYWORD_WORKABLE, secondBucket.get(KEY)); } finally { wipeOfTestResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, null, null, SEARCH_PIPELINE); } @@ -747,7 +749,7 @@ private void testSignificantTermsAggs() throws IOException { try { prepareResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, SEARCH_PIPELINE); - AggregationBuilder aggsBuilder = AggregationBuilders.significantTerms(GENERIC_AGGREGATION_NAME).field(KEYWORD_FIELD_1); + AggregationBuilder aggsBuilder = AggregationBuilders.significantTerms(GENERIC_AGGREGATION_NAME).field(KEYWORD_FIELD_DOCKEYWORD); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS diff --git a/src/test/java/org/opensearch/neuralsearch/query/aggregation/MetricAggregationsWithHybridQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/aggregation/MetricAggregationsWithHybridQueryIT.java index b40e94f2a..5e1b00aba 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/aggregation/MetricAggregationsWithHybridQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/aggregation/MetricAggregationsWithHybridQueryIT.java @@ -185,7 +185,7 @@ private void testAvgAggs() throws IOException { try { prepareResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, SEARCH_PIPELINE); - AggregationBuilder aggsBuilder = AggregationBuilders.avg(AVG_AGGREGATION_NAME).field(INTEGER_FIELD_1); + AggregationBuilder aggsBuilder = AggregationBuilders.avg(AVG_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS @@ -204,7 +204,7 @@ private void testAvgAggs() throws IOException { private void testCardinalityAggs() throws IOException { try { prepareResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, SEARCH_PIPELINE); - AggregationBuilder aggsBuilder = AggregationBuilders.cardinality(GENERIC_AGGREGATION_NAME).field(INTEGER_FIELD_1); + AggregationBuilder aggsBuilder = AggregationBuilders.cardinality(GENERIC_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS @@ -223,7 +223,7 @@ private void testCardinalityAggs() throws IOException { private void testExtendedStatsAggs() throws IOException { try { prepareResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, SEARCH_PIPELINE); - AggregationBuilder aggsBuilder = AggregationBuilders.extendedStats(GENERIC_AGGREGATION_NAME).field(INTEGER_FIELD_1); + AggregationBuilder aggsBuilder = AggregationBuilders.extendedStats(GENERIC_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS @@ -273,7 +273,13 @@ private void testScriptedMetricsAggs() throws IOException { AggregationBuilder aggsBuilder = AggregationBuilders.scriptedMetric(GENERIC_AGGREGATION_NAME) .initScript(new Script("state.price = []")) .mapScript( - new Script("state.price.add(doc[\"" + INTEGER_FIELD_1 + "\"].size() == 0 ? 0 : doc." + INTEGER_FIELD_1 + ".value)") + new Script( + "state.price.add(doc[\"" + + INTEGER_FIELD_DOCINDEX + + "\"].size() == 0 ? 0 : doc." + + INTEGER_FIELD_DOCINDEX + + ".value)" + ) ) .combineScript(new Script("state.price.stream().mapToInt(Integer::intValue).sum()")) .reduceScript(new Script("states.stream().mapToInt(Integer::intValue).sum()")); @@ -295,7 +301,7 @@ private void testScriptedMetricsAggs() throws IOException { private void testPercentileAggs() throws IOException { try { prepareResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, SEARCH_PIPELINE); - AggregationBuilder aggsBuilder = AggregationBuilders.percentiles(GENERIC_AGGREGATION_NAME).field(INTEGER_FIELD_1); + AggregationBuilder aggsBuilder = AggregationBuilders.percentiles(GENERIC_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS @@ -328,7 +334,7 @@ private void testPercentileRankAggs() throws IOException { try { prepareResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, SEARCH_PIPELINE); AggregationBuilder aggsBuilder = AggregationBuilders.percentileRanks(GENERIC_AGGREGATION_NAME, new double[] { 2000, 3000 }) - .field(INTEGER_FIELD_1); + .field(INTEGER_FIELD_DOCINDEX); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS @@ -354,7 +360,7 @@ private void testSumAggs() throws IOException { try { prepareResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, SEARCH_PIPELINE); - AggregationBuilder aggsBuilder = AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_1); + AggregationBuilder aggsBuilder = AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS @@ -373,7 +379,7 @@ private void testSumAggs() throws IOException { private void testValueCountAggs() throws IOException { try { prepareResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, SEARCH_PIPELINE); - AggregationBuilder aggsBuilder = AggregationBuilders.count(GENERIC_AGGREGATION_NAME).field(INTEGER_FIELD_1); + AggregationBuilder aggsBuilder = AggregationBuilders.count(GENERIC_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX); Map searchResponseAsMap = executeQueryAndGetAggsResults( aggsBuilder, TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS @@ -395,7 +401,7 @@ private void testSumAggsAndRangePostFilter() throws IOException { try { prepareResources(TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, SEARCH_PIPELINE); - AggregationBuilder aggsBuilder = AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_1); + AggregationBuilder aggsBuilder = AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX); TermQueryBuilder termQueryBuilder1 = QueryBuilders.termQuery(TEST_TEXT_FIELD_NAME_1, TEST_QUERY_TEXT3); TermQueryBuilder termQueryBuilder2 = QueryBuilders.termQuery(TEST_TEXT_FIELD_NAME_1, TEST_QUERY_TEXT4); @@ -406,7 +412,7 @@ private void testSumAggsAndRangePostFilter() throws IOException { hybridQueryBuilderNeuralThenTerm.add(termQueryBuilder2); hybridQueryBuilderNeuralThenTerm.add(termQueryBuilder3); - QueryBuilder rangeFilterQuery = QueryBuilders.rangeQuery(INTEGER_FIELD_1).gte(3000).lte(5000); + QueryBuilder rangeFilterQuery = QueryBuilders.rangeQuery(INTEGER_FIELD_DOCINDEX).gte(3000).lte(5000); Map searchResponseAsMap = search( TEST_MULTI_DOC_INDEX_WITH_TEXT_AND_INT_MULTIPLE_SHARDS, @@ -433,7 +439,7 @@ private void testSumAggsAndRangePostFilter() throws IOException { for (Map oneHit : hitsNestedList) { assertNotNull(oneHit.get("_source")); Map source = (Map) oneHit.get("_source"); - int docIndex = (int) source.get(INTEGER_FIELD_1); + int docIndex = (int) source.get(INTEGER_FIELD_DOCINDEX); docIndexes.add(docIndex); } assertEquals(0, docIndexes.stream().filter(docIndex -> docIndex < 3000 || docIndex > 5000).count()); diff --git a/src/test/java/org/opensearch/neuralsearch/query/aggregation/PipelineAggregationsWithHybridQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/aggregation/PipelineAggregationsWithHybridQueryIT.java index 2e41d6e96..a83954510 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/aggregation/PipelineAggregationsWithHybridQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/aggregation/PipelineAggregationsWithHybridQueryIT.java @@ -105,8 +105,8 @@ private void testDateBucketedSumsPipelinedToBucketStatsAggs() throws IOException AggregationBuilder aggDateHisto = AggregationBuilders.dateHistogram(GENERIC_AGGREGATION_NAME) .calendarInterval(DateHistogramInterval.YEAR) - .field(DATE_FIELD_1) - .subAggregation(AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_1)); + .field(DATE_FIELD) + .subAggregation(AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX)); StatsBucketPipelineAggregationBuilder aggStatsBucket = PipelineAggregatorBuilders.statsBucket( BUCKETS_AGGREGATION_NAME_1, @@ -141,19 +141,21 @@ private void testDateBucketedSumsPipelinedToBucketScriptedAggs() throws IOExcept AggregationBuilder aggBuilder = AggregationBuilders.dateHistogram(DATE_AGGREGATION_NAME) .calendarInterval(DateHistogramInterval.YEAR) - .field(DATE_FIELD_1) + .field(DATE_FIELD) .subAggregations( - new AggregatorFactories.Builder().addAggregator(AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_1)) + new AggregatorFactories.Builder().addAggregator( + AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX) + ) .addAggregator( AggregationBuilders.filter( GENERIC_AGGREGATION_NAME, QueryBuilders.boolQuery() .should( QueryBuilders.boolQuery() - .should(QueryBuilders.termQuery(KEYWORD_FIELD_1, KEYWORD_FIELD_1_VALUE)) - .should(QueryBuilders.termQuery(KEYWORD_FIELD_1, KEYWORD_FIELD_2_VALUE)) + .should(QueryBuilders.termQuery(KEYWORD_FIELD_DOCKEYWORD, KEYWORD_FIELD_DOCKEYWORD_WORKABLE)) + .should(QueryBuilders.termQuery(KEYWORD_FIELD_DOCKEYWORD, KEYWORD_FIELD_DOCKEYWORD_ANGRY)) ) - .should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(KEYWORD_FIELD_1))) + .should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(KEYWORD_FIELD_DOCKEYWORD))) ).subAggregation(AggregationBuilders.sum(SUM_AGGREGATION_NAME_2).field(INTEGER_FIELD_PRICE)) ) .addPipelineAggregator( @@ -231,9 +233,11 @@ private void testDateBucketedSumsPipelinedToBucketSortAggs() throws IOException AggregationBuilder aggBuilder = AggregationBuilders.dateHistogram(DATE_AGGREGATION_NAME) .calendarInterval(DateHistogramInterval.YEAR) - .field(DATE_FIELD_1) + .field(DATE_FIELD) .subAggregations( - new AggregatorFactories.Builder().addAggregator(AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_1)) + new AggregatorFactories.Builder().addAggregator( + AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX) + ) .addPipelineAggregator( PipelineAggregatorBuilders.bucketSort( BUCKETS_AGGREGATION_NAME_1, @@ -245,10 +249,10 @@ private void testDateBucketedSumsPipelinedToBucketSortAggs() throws IOException QueryBuilder queryBuilder = QueryBuilders.boolQuery() .should( QueryBuilders.boolQuery() - .should(QueryBuilders.termQuery(KEYWORD_FIELD_1, KEYWORD_FIELD_1_VALUE)) - .should(QueryBuilders.termQuery(KEYWORD_FIELD_1, KEYWORD_FIELD_2_VALUE)) + .should(QueryBuilders.termQuery(KEYWORD_FIELD_DOCKEYWORD, KEYWORD_FIELD_DOCKEYWORD_WORKABLE)) + .should(QueryBuilders.termQuery(KEYWORD_FIELD_DOCKEYWORD, KEYWORD_FIELD_DOCKEYWORD_ANGRY)) ) - .should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(KEYWORD_FIELD_1))); + .should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(KEYWORD_FIELD_DOCKEYWORD))); Map searchResponseAsMap = executeQueryAndGetAggsResults( List.of(aggBuilder), @@ -298,19 +302,20 @@ private void testDateBucketedSumsPipelinedToCumulativeSumAggs() throws IOExcepti AggregationBuilder aggBuilder = AggregationBuilders.dateHistogram(DATE_AGGREGATION_NAME) .calendarInterval(DateHistogramInterval.YEAR) - .field(DATE_FIELD_1) + .field(DATE_FIELD) .subAggregations( - new AggregatorFactories.Builder().addAggregator(AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_1)) - .addPipelineAggregator(PipelineAggregatorBuilders.cumulativeSum(BUCKETS_AGGREGATION_NAME_1, SUM_AGGREGATION_NAME)) + new AggregatorFactories.Builder().addAggregator( + AggregationBuilders.sum(SUM_AGGREGATION_NAME).field(INTEGER_FIELD_DOCINDEX) + ).addPipelineAggregator(PipelineAggregatorBuilders.cumulativeSum(BUCKETS_AGGREGATION_NAME_1, SUM_AGGREGATION_NAME)) ); QueryBuilder queryBuilder = QueryBuilders.boolQuery() .should( QueryBuilders.boolQuery() - .should(QueryBuilders.termQuery(KEYWORD_FIELD_1, KEYWORD_FIELD_1_VALUE)) - .should(QueryBuilders.termQuery(KEYWORD_FIELD_1, KEYWORD_FIELD_2_VALUE)) + .should(QueryBuilders.termQuery(KEYWORD_FIELD_DOCKEYWORD, KEYWORD_FIELD_DOCKEYWORD_WORKABLE)) + .should(QueryBuilders.termQuery(KEYWORD_FIELD_DOCKEYWORD, KEYWORD_FIELD_DOCKEYWORD_ANGRY)) ) - .should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(KEYWORD_FIELD_1))); + .should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(KEYWORD_FIELD_DOCKEYWORD))); Map searchResponseAsMap = executeQueryAndGetAggsResults( List.of(aggBuilder),