From 697fcf32102f8ad27701c9473b6371269fd2eeb0 Mon Sep 17 00:00:00 2001 From: Ganesh Krishna Ramadurai Date: Thu, 16 Nov 2023 09:59:44 -0800 Subject: [PATCH] Fix tests for concurrent search with IndexRandomForConcurrentSearch follow-up [#11130] (#11212) Signed-off-by: Ganesh Ramadurai Co-authored-by: Ganesh Ramadurai --- .../search/scroll/SearchScrollIT.java | 14 ++++++++-- .../opensearch/search/sort/FieldSortIT.java | 28 +++++++++++++++---- .../opensearch/search/sort/GeoDistanceIT.java | 3 ++ .../search/source/SourceFetchingIT.java | 6 ++-- .../search/suggest/SuggestSearchIT.java | 1 + 5 files changed, 43 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java index 0eee136acac69..295a6e3441401 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java @@ -135,6 +135,7 @@ public void testSimpleScrollQueryThenFetch() throws Exception { } client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) @@ -188,6 +189,7 @@ public void testSimpleScrollQueryThenFetchSmallSizeUnevenDistribution() throws E } client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setSearchType(SearchType.QUERY_THEN_FETCH) @@ -256,6 +258,7 @@ public void testScrollAndUpdateIndex() throws Exception { } client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); assertThat(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get().getHits().getTotalHits().value, equalTo(500L)); assertThat( @@ -328,6 +331,7 @@ public void testSimpleScrollQueryThenFetch_clearScrollIds() throws Exception { } client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse1 = client().prepareSearch() .setQuery(matchAllQuery()) @@ -448,6 +452,7 @@ public void testSimpleScrollQueryThenFetchClearAllScrollIds() throws Exception { } client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse1 = client().prepareSearch() .setQuery(matchAllQuery()) @@ -526,6 +531,7 @@ public void testDeepScrollingDoesNotBlowUp() throws Exception { .prepareUpdateSettings("index") .setSettings(Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE)) .get(); + indexRandomForConcurrentSearch("index"); for (SearchType searchType : SearchType.values()) { SearchRequestBuilder builder = client().prepareSearch("index") @@ -567,6 +573,7 @@ public void testStringSortMissingAscTerminates() throws Exception { ); client().prepareIndex("test").setId("1").setSource("some_field", "test").get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch("test") .addSort(new FieldSortBuilder("no_field").order(SortOrder.ASC).missing("_last")) @@ -592,12 +599,13 @@ public void testStringSortMissingAscTerminates() throws Exception { assertThat(response.getHits().getHits().length, equalTo(0)); } - public void testCloseAndReopenOrDeleteWithActiveScroll() { + public void testCloseAndReopenOrDeleteWithActiveScroll() throws InterruptedException { createIndex("test"); for (int i = 0; i < 100; i++) { client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", i).get(); } refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .setSize(35) @@ -683,7 +691,7 @@ public void testScrollInvalidDefaultKeepAlive() throws IOException { assertThat(exc.getMessage(), containsString("was (1m > 30s)")); } - public void testInvalidScrollKeepAlive() throws IOException { + public void testInvalidScrollKeepAlive() throws IOException, InterruptedException { createIndex("test"); for (int i = 0; i < 2; i++) { client().prepareIndex("test") @@ -692,6 +700,7 @@ public void testInvalidScrollKeepAlive() throws IOException { .get(); } refresh(); + indexRandomForConcurrentSearch("test"); assertAcked( client().admin() .cluster() @@ -793,6 +802,7 @@ public void testRestartDataNodesDuringScrollSearch() throws Exception { index("prod", "_doc", "prod-" + i, Collections.emptyMap()); } client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("demo", "prod"); SearchResponse respFromDemoIndex = client().prepareSearch("demo") .setSize(randomIntBetween(1, 10)) .setQuery(new MatchAllQueryBuilder()) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java index d4980a64a3977..1e2a8e246a26c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java @@ -154,7 +154,7 @@ protected Collection> nodePlugins() { return Arrays.asList(InternalSettingsPlugin.class, CustomScriptPlugin.class); } - public void testIssue8226() { + public void testIssue8226() throws InterruptedException { int numIndices = between(5, 10); final boolean useMapping = randomBoolean(); for (int i = 0; i < numIndices; i++) { @@ -168,6 +168,9 @@ public void testIssue8226() { } } refresh(); + for (int i = 0; i < numIndices; i++) { + indexRandomForConcurrentSearch("test_" + i); + } // sort DESC SearchResponse searchResponse = client().prepareSearch() .addSort(new FieldSortBuilder("entry").order(SortOrder.DESC).unmappedType(useMapping ? null : "long")) @@ -446,6 +449,7 @@ public void testScoreSortDirection() throws Exception { client().prepareIndex("test").setId("3").setSource("field", 0).get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.fieldValueFactorFunction("field"))) @@ -484,6 +488,7 @@ public void testScoreSortDirectionWithFunctionScore() throws Exception { client().prepareIndex("test").setId("3").setSource("field", 0).get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch("test") .setQuery(functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("field"))) @@ -531,7 +536,7 @@ public void testIssue2986() { } } - public void testIssue2991() { + public void testIssue2991() throws InterruptedException { for (int i = 1; i < 4; i++) { try { client().admin().indices().prepareDelete("test").get(); @@ -552,6 +557,7 @@ public void testIssue2991() { client().prepareIndex("test").setId("2").setSource("tag", "beta").get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse resp = client().prepareSearch("test") .setSize(2) .setQuery(matchAllQuery()) @@ -650,6 +656,7 @@ public void testSimpleSorts() throws Exception { } refresh(); + indexRandomForConcurrentSearch("test"); // STRING int size = 1 + random.nextInt(10); @@ -922,6 +929,7 @@ public void testSortMissingNumbers() throws Exception { flush(); refresh(); + indexRandomForConcurrentSearch("test"); // DOUBLE logger.info("--> sort with no missing (same as missing _last)"); @@ -1083,6 +1091,7 @@ public void testSortMissingNumbersMinMax() throws Exception { flush(); refresh(); + indexRandomForConcurrentSearch("test"); // LONG logger.info("--> sort with no missing (same as missing _last)"); @@ -1202,7 +1211,7 @@ public void testSortMissingNumbersMinMax() throws Exception { assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); } - public void testSortMissingStrings() throws IOException { + public void testSortMissingStrings() throws IOException, InterruptedException { assertAcked( prepareCreate("test").setMapping( XContentFactory.jsonBuilder() @@ -1230,6 +1239,7 @@ public void testSortMissingStrings() throws IOException { flush(); refresh(); + indexRandomForConcurrentSearch("test"); // TODO: WTF? try { @@ -1418,6 +1428,7 @@ public void testSortMVField() throws Exception { .get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) @@ -1702,7 +1713,7 @@ public void testSortMVField() throws Exception { assertThat(searchResponse.getHits().getAt(2).getSortValues()[0], equalTo("03")); } - public void testSortOnRareField() throws IOException { + public void testSortOnRareField() throws IOException, InterruptedException { assertAcked( prepareCreate("test").setMapping( XContentFactory.jsonBuilder() @@ -1722,6 +1733,7 @@ public void testSortOnRareField() throws IOException { .get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .setSize(3) @@ -1817,6 +1829,7 @@ public void testSortMetaField() throws Exception { indexReqs[i] = client().prepareIndex("test").setId(Integer.toString(i)).setSource(); } indexRandom(true, indexReqs); + indexRandomForConcurrentSearch("test"); SortOrder order = randomFrom(SortOrder.values()); SearchResponse searchResponse = client().prepareSearch() @@ -1919,6 +1932,7 @@ public void testNestedSort() throws IOException, InterruptedException, Execution ) .get(); refresh(); + indexRandomForConcurrentSearch("test"); // We sort on nested field SearchResponse searchResponse = client().prepareSearch() @@ -2104,6 +2118,7 @@ public void testFieldAlias() throws Exception { builders.add(client().prepareIndex("old_index").setSource("distance", 50.5)); builders.add(client().prepareIndex("new_index").setSource("route_length_miles", 100.2)); indexRandom(true, true, builders); + indexRandomForConcurrentSearch("old_index", "new_index"); SearchResponse response = client().prepareSearch() .setQuery(matchAllQuery()) @@ -2130,6 +2145,7 @@ public void testFieldAliasesWithMissingValues() throws Exception { builders.add(client().prepareIndex("old_index").setSource(Collections.emptyMap())); builders.add(client().prepareIndex("new_index").setSource("route_length_miles", 100.2)); indexRandom(true, true, builders); + indexRandomForConcurrentSearch("old_index", "new_index"); SearchResponse response = client().prepareSearch() .setQuery(matchAllQuery()) @@ -2199,6 +2215,7 @@ public void testCastDate() throws Exception { builders.add(client().prepareIndex("index_date").setSource("field", "2024-04-11T23:47:17")); builders.add(client().prepareIndex("index_date_nanos").setSource("field", "2024-04-11T23:47:16.854775807Z")); indexRandom(true, true, builders); + indexRandomForConcurrentSearch("index_date", "index_date_nanos"); { SearchResponse response = client().prepareSearch() @@ -2324,7 +2341,7 @@ public void testCastNumericTypeExceptions() throws Exception { } } - public void testLongSortOptimizationCorrectResults() { + public void testLongSortOptimizationCorrectResults() throws InterruptedException { assertAcked( prepareCreate("test1").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)) .setMapping("long_field", "type=long") @@ -2341,6 +2358,7 @@ public void testLongSortOptimizationCorrectResults() { bulkBuilder.add(client().prepareIndex("test1").setId(Integer.toString(i)).setSource(source, MediaTypeRegistry.JSON)); } refresh(); + indexRandomForConcurrentSearch("test1"); // *** 1. sort DESC on long_field SearchResponse searchResponse = client().prepareSearch() diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java index 6886f8d67589e..aeb16f5ce9294 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java @@ -192,6 +192,7 @@ public void testDistanceSortingMVFields() throws Exception { .get(); client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); // Order: Asc SearchResponse searchResponse = client().prepareSearch("test") @@ -324,6 +325,7 @@ public void testDistanceSortingWithMissingGeoPoint() throws Exception { .get(); refresh(); + indexRandomForConcurrentSearch("test"); // Order: Asc SearchResponse searchResponse = client().prepareSearch("test") @@ -668,6 +670,7 @@ public void testDistanceSortingWithUnmappedField() throws Exception { .get(); refresh(); + indexRandomForConcurrentSearch("test1", "test2"); // Order: Asc SearchResponse searchResponse = client().prepareSearch("test1", "test2") diff --git a/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java index eeef5403fe898..f3b9c6daf8988 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java @@ -84,12 +84,13 @@ public void testSourceDefaultBehavior() { } - public void testSourceFiltering() { + public void testSourceFiltering() throws InterruptedException { createIndex("test"); ensureGreen(); client().prepareIndex("test").setId("1").setSource("field1", "value", "field2", "value2").get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch("test").setFetchSource(false).get(); assertThat(response.getHits().getAt(0).getSourceAsString(), nullValue()); @@ -117,12 +118,13 @@ public void testSourceFiltering() { * Test Case for #5132: Source filtering with wildcards broken when given multiple patterns * https://github.com/elastic/elasticsearch/issues/5132 */ - public void testSourceWithWildcardFiltering() { + public void testSourceWithWildcardFiltering() throws InterruptedException { createIndex("test"); ensureGreen(); client().prepareIndex("test").setId("1").setSource("field", "value").get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch("test").setFetchSource(new String[] { "*.notexisting", "field" }, null).get(); assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java index 32bb0e34054bb..7a72a92dc98fc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java @@ -369,6 +369,7 @@ public void testSimple() throws Exception { index("test", "type1", "3", "text", "abbd"); index("test", "type1", "4", "text", "abcc"); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "spellcecker")).get(); assertThat("didn't ask for suggestions but got some", search.getSuggest(), nullValue());