[Backport 2.x] Add indexRandomForConcurrentSearch for tests

server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java

@@ -169,7 +169,7 @@ private Settings.Builder settingsBuilder() {
         return Settings.builder().put(indexSettings());
     }
 
-    public void testFieldDataStats() {
+    public void testFieldDataStats() throws InterruptedException {
         assertAcked(
             client().admin()
                 .indices()
@@ -182,6 +182,7 @@ public void testFieldDataStats() {
         client().prepareIndex("test").setId("1").setSource("field", "value1", "field2", "value1").execute().actionGet();
         client().prepareIndex("test").setId("2").setSource("field", "value2", "field2", "value2").execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
+        indexRandomForConcurrentSearch("test");
 
         NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
         assertThat(
@@ -305,6 +306,7 @@ public void testClearAllCaches() throws Exception {
         client().prepareIndex("test").setId("1").setSource("field", "value1").execute().actionGet();
         client().prepareIndex("test").setId("2").setSource("field", "value2").execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
+        indexRandomForConcurrentSearch("test");
 
         NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
         assertThat(

server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java

@@ -93,6 +93,7 @@ public void testSimpleTimeout() throws Exception {
             client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get();
         }
         refresh("test");
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test")
             .setTimeout(new TimeValue(5, TimeUnit.MILLISECONDS))
@@ -104,12 +105,11 @@ public void testSimpleTimeout() throws Exception {
     }
 
     public void testSimpleDoesNotTimeout() throws Exception {
-        final int numDocs = 10;
+        final int numDocs = 9;
         for (int i = 0; i < numDocs; i++) {
-            client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").get();
+            client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         }
-        refresh("test");
-
+        indexRandomForConcurrentSearch("test");
         SearchResponse searchResponse = client().prepareSearch("test")
             .setTimeout(new TimeValue(10000, TimeUnit.SECONDS))
             .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap())))
@@ -122,7 +122,7 @@ public void testSimpleDoesNotTimeout() throws Exception {
 
     public void testPartialResultsIntolerantTimeout() throws Exception {
         client().prepareIndex("test").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-
+        indexRandomForConcurrentSearch("test");
         OpenSearchException ex = expectThrows(
             OpenSearchException.class,
             () -> client().prepareSearch("test")

server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NaNSortingIT.java

@@ -172,6 +172,7 @@ public void setupSuiteScopeCluster() throws Exception {
             client().prepareIndex("idx").setSource(source.endObject()).get();
         }
         refresh();
+        indexRandomForMultipleSlices("idx");
         ensureSearchable();
     }
 

server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/NestedIT.java

@@ -224,6 +224,7 @@ public void setupSuiteScopeCluster() throws Exception {
                 )
         );
         indexRandom(true, builders);
+        indexRandomForMultipleSlices("idx");
         ensureSearchable();
     }
 
@@ -354,6 +355,7 @@ public void testNestedAsSubAggregation() throws Exception {
     }
 
     public void testNestNestedAggs() throws Exception {
+        indexRandomForConcurrentSearch("idx_nested_nested_aggs");
         SearchResponse response = client().prepareSearch("idx_nested_nested_aggs")
             .addAggregation(
                 nested("level1", "nested1").subAggregation(
@@ -607,6 +609,7 @@ public void testNestedSameDocIdProcessedMultipleTime() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("idx4");
 
         SearchResponse response = client().prepareSearch("idx4")
             .addAggregation(
@@ -782,6 +785,7 @@ public void testFilterAggInsideNestedAgg() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("classes");
 
         SearchResponse response = client().prepareSearch("classes")
             .addAggregation(

server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/RangeIT.java

@@ -184,6 +184,7 @@ public void setupSuiteScopeCluster() throws Exception {
         builders.add(client().prepareIndex("new_index").setSource(Collections.emptyMap()));
 
         indexRandom(true, builders);
+        indexRandomForMultipleSlices("idx", "old_index", "new_index");
         ensureSearchable();
     }
 
@@ -917,6 +918,7 @@ public void testOverlappingRanges() throws Exception {
     }
 
     public void testEmptyAggregation() throws Exception {
+        indexRandomForConcurrentSearch("empty_bucket_idx");
         SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx")
             .setQuery(matchAllQuery())
             .addAggregation(

server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java

@@ -100,10 +100,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
 
     @SuppressWarnings("unchecked")
     public void testPlugin() throws Exception {
-        assumeFalse(
-            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11112",
-            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
-        );
         client().admin()
             .indices()
             .prepareCreate("test")

server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/InnerHitsIT.java

@@ -897,6 +897,7 @@ public void testNestedSource() throws Exception {
             )
             .get();
         refresh();
+        indexRandomForConcurrentSearch("index1");
 
         // the field name (comments.message) used for source filtering should be the same as when using that field for
         // other features (like in the query dsl or aggs) in order for consistency:
@@ -973,6 +974,7 @@ public void testInnerHitsWithIgnoreUnmapped() throws Exception {
         client().prepareIndex("index1").setId("1").setSource("nested_type", Collections.singletonMap("key", "value")).get();
         client().prepareIndex("index2").setId("3").setSource("key", "value").get();
         refresh();
+        indexRandomForConcurrentSearch("index1", "index2");
 
         SearchResponse response = client().prepareSearch("index1", "index2")
             .setQuery(
@@ -1002,6 +1004,7 @@ public void testUseMaxDocInsteadOfSize() throws Exception {
             .setRefreshPolicy(IMMEDIATE)
             .get();
 
+        indexRandomForConcurrentSearch("index2");
         QueryBuilder query = nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg).innerHit(
             new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1)
         );
@@ -1019,6 +1022,7 @@ public void testTooHighResultWindow() throws Exception {
             )
             .setRefreshPolicy(IMMEDIATE)
             .get();
+        indexRandomForConcurrentSearch("index2");
         SearchResponse response = client().prepareSearch("index2")
             .setQuery(
                 nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg).innerHit(

server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/MatchedQueriesIT.java

@@ -91,6 +91,7 @@ public void testSimpleMatchedQueryFromFilteredQuery() throws Exception {
         client().prepareIndex("test").setId("2").setSource("name", "test2", "number", 2).get();
         client().prepareIndex("test").setId("3").setSource("name", "test3", "number", 3).get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(
@@ -141,6 +142,7 @@ public void testSimpleMatchedQueryFromTopLevelFilter() throws Exception {
         client().prepareIndex("test").setId("2").setSource("name", "test").get();
         client().prepareIndex("test").setId("3").setSource("name", "test").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(matchAllQuery())
@@ -192,6 +194,7 @@ public void testSimpleMatchedQueryFromTopLevelFilterAndFilteredQuery() throws Ex
         client().prepareIndex("test").setId("2").setSource("name", "test", "title", "title2").get();
         client().prepareIndex("test").setId("3").setSource("name", "test", "title", "title3").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(boolQuery().must(matchAllQuery()).filter(termsQuery("title", "title1", "title2", "title3").queryName("title")))
@@ -224,12 +227,13 @@ public void testSimpleMatchedQueryFromTopLevelFilterAndFilteredQuery() throws Ex
         }
     }
 
-    public void testRegExpQuerySupportsName() {
+    public void testRegExpQuerySupportsName() throws InterruptedException {
         createIndex("test1");
         ensureGreen();
 
         client().prepareIndex("test1").setId("1").setSource("title", "title1").get();
         refresh();
+        indexRandomForConcurrentSearch("test1");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(QueryBuilders.regexpQuery("title", "title1").queryName("regex"))
@@ -246,12 +250,13 @@ public void testRegExpQuerySupportsName() {
         }
     }
 
-    public void testPrefixQuerySupportsName() {
+    public void testPrefixQuerySupportsName() throws InterruptedException {
         createIndex("test1");
         ensureGreen();
 
         client().prepareIndex("test1").setId("1").setSource("title", "title1").get();
         refresh();
+        indexRandomForConcurrentSearch("test1");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(QueryBuilders.prefixQuery("title", "title").queryName("prefix"))
@@ -268,12 +273,13 @@ public void testPrefixQuerySupportsName() {
         }
     }
 
-    public void testFuzzyQuerySupportsName() {
+    public void testFuzzyQuerySupportsName() throws InterruptedException {
         createIndex("test1");
         ensureGreen();
 
         client().prepareIndex("test1").setId("1").setSource("title", "title1").get();
         refresh();
+        indexRandomForConcurrentSearch("test1");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(QueryBuilders.fuzzyQuery("title", "titel1").queryName("fuzzy"))
@@ -290,12 +296,13 @@ public void testFuzzyQuerySupportsName() {
         }
     }
 
-    public void testWildcardQuerySupportsName() {
+    public void testWildcardQuerySupportsName() throws InterruptedException {
         createIndex("test1");
         ensureGreen();
 
         client().prepareIndex("test1").setId("1").setSource("title", "title1").get();
         refresh();
+        indexRandomForConcurrentSearch("test1");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(QueryBuilders.wildcardQuery("title", "titl*").queryName("wildcard"))
@@ -312,12 +319,13 @@ public void testWildcardQuerySupportsName() {
         }
     }
 
-    public void testSpanFirstQuerySupportsName() {
+    public void testSpanFirstQuerySupportsName() throws InterruptedException {
         createIndex("test1");
         ensureGreen();
 
         client().prepareIndex("test1").setId("1").setSource("title", "title1 title2").get();
         refresh();
+        indexRandomForConcurrentSearch("test1");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(QueryBuilders.spanFirstQuery(QueryBuilders.spanTermQuery("title", "title1"), 10).queryName("span"))
@@ -344,6 +352,7 @@ public void testMatchedWithShould() throws Exception {
         client().prepareIndex("test").setId("1").setSource("content", "Lorem ipsum dolor sit amet").get();
         client().prepareIndex("test").setId("2").setSource("content", "consectetur adipisicing elit").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         // Execute search at least two times to load it in cache
         int iter = scaledRandomIntBetween(2, 10);
@@ -378,6 +387,7 @@ public void testMatchedWithWrapperQuery() throws Exception {
 
         client().prepareIndex("test").setId("1").setSource("content", "Lorem ipsum dolor sit amet").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         MatchQueryBuilder matchQueryBuilder = matchQuery("content", "amet").queryName("abc");
         BytesReference matchBytes = XContentHelper.toXContent(matchQueryBuilder, MediaTypeRegistry.JSON, false);

server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java

@@ -135,6 +135,7 @@ public void testConsistentHitsWithSameSeed() throws Exception {
         }
         flush();
         refresh();
+        indexRandomForConcurrentSearch("test");
         int outerIters = scaledRandomIntBetween(10, 20);
         for (int o = 0; o < outerIters; o++) {
             final int seed = randomInt();
@@ -299,6 +300,7 @@ public void testSeedReportedInExplain() throws Exception {
         index("test", "type", "1", jsonBuilder().startObject().endObject());
         flush();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         int seed = 12345678;
 
@@ -318,6 +320,7 @@ public void testSeedAndNameReportedInExplain() throws Exception {
         index("test", "type", "1", jsonBuilder().startObject().endObject());
         flush();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         int seed = 12345678;
 

server/src/internalClusterTest/java/org/opensearch/search/msearch/MultiSearchIT.java

@@ -71,12 +71,13 @@ protected Settings featureFlagSettings() {
         return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
     }
 
-    public void testSimpleMultiSearch() {
+    public void testSimpleMultiSearch() throws InterruptedException {
         createIndex("test");
         ensureGreen();
         client().prepareIndex("test").setId("1").setSource("field", "xxx").get();
         client().prepareIndex("test").setId("2").setSource("field", "yyy").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
         MultiSearchResponse response = client().prepareMultiSearch()
             .add(client().prepareSearch("test").setQuery(QueryBuilders.termQuery("field", "xxx")))
             .add(client().prepareSearch("test").setQuery(QueryBuilders.termQuery("field", "yyy")))
@@ -94,13 +95,14 @@ public void testSimpleMultiSearch() {
         assertFirstHit(response.getResponses()[1].getResponse(), hasId("2"));
     }
 
-    public void testSimpleMultiSearchMoreRequests() {
+    public void testSimpleMultiSearchMoreRequests() throws InterruptedException {
         createIndex("test");
         int numDocs = randomIntBetween(0, 16);
         for (int i = 0; i < numDocs; i++) {
             client().prepareIndex("test").setId(Integer.toString(i)).setSource("{}", MediaTypeRegistry.JSON).get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         int numSearchRequests = randomIntBetween(1, 64);
         MultiSearchRequest request = new MultiSearchRequest();