From f54ada3c49c4c5c4fd1ff6211ba9dbda87b5e92b Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Mon, 4 Sep 2023 03:00:13 -0400 Subject: [PATCH] Add average query concurrency metric for concurrent segment search (#9670) Signed-off-by: Jay Deng --- CHANGELOG.md | 1 + .../test/cat.shards/10_basic.yml | 96 +-------- .../search/stats/ConcurrentSearchStatsIT.java | 200 +++++++++++++++++- .../index/search/stats/SearchStats.java | 26 ++- .../index/search/stats/ShardSearchStats.java | 4 + .../rest/action/cat/RestIndicesAction.java | 56 +++-- .../rest/action/cat/RestNodesAction.java | 40 ++-- .../rest/action/cat/RestShardsAction.java | 41 ++-- .../index/search/stats/SearchStatsTests.java | 8 +- .../action/cat/RestShardsActionTests.java | 4 +- 10 files changed, 313 insertions(+), 163 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c875eca04b13..e583104e97ebc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Expose DelimitedTermFrequencyTokenFilter to allow providing term frequencies along with terms ([#9479](https://github.com/opensearch-project/OpenSearch/pull/9479)) - APIs for performing async blob reads and async downloads from the repository using multiple streams ([#9592](https://github.com/opensearch-project/OpenSearch/issues/9592)) - Add concurrent segment search related metrics to node and index stats ([#9622](https://github.com/opensearch-project/OpenSearch/issues/9622)) +- Add average concurrency metric for concurrent segment search ([#9670](https://github.com/opensearch-project/OpenSearch/issues/9670)) ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml index 88896e3907678..f80c9f9c0bc80 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml @@ -1,105 +1,13 @@ "Help": - skip: - version: " - 2.9.99" - reason: concurrent search stats were added in 2.10.0 - features: node_selector - - do: - cat.shards: - help: true - node_selector: - version: "2.10.0 - " - - - match: - $body: | - /^ index .+ \n - shard .+ \n - prirep .+ \n - state .+ \n - docs .+ \n - store .+ \n - ip .+ \n - id .+ \n - node .+ \n - sync_id .+ \n - unassigned.reason .+ \n - unassigned.at .+ \n - unassigned.for .+ \n - unassigned.details .+ \n - recoverysource.type .+ \n - completion.size .+ \n - fielddata.memory_size .+ \n - fielddata.evictions .+ \n - query_cache.memory_size .+ \n - query_cache.evictions .+ \n - flush.total .+ \n - flush.total_time .+ \n - get.current .+ \n - get.time .+ \n - get.total .+ \n - get.exists_time .+ \n - get.exists_total .+ \n - get.missing_time .+ \n - get.missing_total .+ \n - indexing.delete_current .+ \n - indexing.delete_time .+ \n - indexing.delete_total .+ \n - indexing.index_current .+ \n - indexing.index_time .+ \n - indexing.index_total .+ \n - indexing.index_failed .+ \n - merges.current .+ \n - merges.current_docs .+ \n - merges.current_size .+ \n - merges.total .+ \n - merges.total_docs .+ \n - merges.total_size .+ \n - merges.total_time .+ \n - refresh.total .+ \n - refresh.time .+ \n - refresh.external_total .+ \n - refresh.external_time .+ \n - refresh.listeners .+ \n - search.fetch_current .+ \n - search.fetch_time .+ \n - search.fetch_total .+ \n - search.open_contexts .+ \n - search.query_current .+ \n - search.query_time .+ \n - search.query_total .+ \n - search.concurrent_query_current .+ \n - search.concurrent_query_time .+ \n - search.concurrent_query_total .+ \n - search.scroll_current .+ \n - search.scroll_time .+ \n - search.scroll_total .+ \n - search.point_in_time_current .+ \n - search.point_in_time_time .+ \n - search.point_in_time_total .+ \n - segments.count .+ \n - segments.memory .+ \n - segments.index_writer_memory .+ \n - segments.version_map_memory .+ \n - segments.fixed_bitset_memory .+ \n - seq_no.max .+ \n - seq_no.local_checkpoint .+ \n - seq_no.global_checkpoint .+ \n - warmer.current .+ \n - warmer.total .+ \n - warmer.total_time .+ \n - path.data .+ \n - path.state .+ \n - $/ ---- -"Help between 2.4.0 - 2.9.99": - - skip: - version: " - 2.3.99 , 2.9.99 - " + version: " - 2.3.99" reason: point in time stats were added in 2.4.0 features: node_selector - do: cat.shards: help: true node_selector: - version: "2.4.0 - 2.10.0" + version: "2.4.0 - " - match: $body: | diff --git a/server/src/internalClusterTest/java/org/opensearch/search/stats/ConcurrentSearchStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/stats/ConcurrentSearchStatsIT.java index 352fe78b2680d..d2af7fc6effe7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/stats/ConcurrentSearchStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/stats/ConcurrentSearchStatsIT.java @@ -8,6 +8,8 @@ package org.opensearch.search.stats; +import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.opensearch.action.admin.indices.stats.IndexStats; import org.opensearch.action.admin.indices.stats.IndicesStatsRequestBuilder; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.cluster.metadata.IndexMetadata; @@ -27,16 +29,20 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Locale; import java.util.Map; import java.util.function.Function; import static org.opensearch.index.query.QueryBuilders.scriptQuery; +import static org.opensearch.search.SearchBootstrapSettings.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_KEY; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 2, numClientNodes = 0) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false) public class ConcurrentSearchStatsIT extends OpenSearchIntegTestCase { + private final int SEGMENT_SLICE_COUNT = 4; + @Override protected Collection> nodePlugins() { return Arrays.asList(ScriptedDelayedPlugin.class, InternalSettingsPlugin.class); @@ -49,6 +55,7 @@ protected Settings nodeSettings(int nodeOrdinal) { .put(super.nodeSettings(nodeOrdinal)) .put(IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), "1ms") .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true) + .put(CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_KEY, SEGMENT_SLICE_COUNT) .build(); } @@ -68,9 +75,11 @@ protected Settings featureFlagSettings() { } public void testConcurrentQueryCount() throws Exception { + String INDEX_1 = "test-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + String INDEX_2 = "test-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); int NUM_SHARDS = randomIntBetween(1, 5); createIndex( - "test1", + INDEX_1, Settings.builder() .put(indexSettings()) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS) @@ -78,7 +87,7 @@ public void testConcurrentQueryCount() throws Exception { .build() ); createIndex( - "test2", + INDEX_2, Settings.builder() .put(indexSettings()) .put("search.concurrent_segment_search.enabled", false) @@ -92,24 +101,24 @@ public void testConcurrentQueryCount() throws Exception { indexRandom( false, true, - client().prepareIndex("test1").setId("1").setSource("foo", "bar"), - client().prepareIndex("test1").setId("2").setSource("foo", "baz"), - client().prepareIndex("test2").setId("1").setSource("foo", "bar"), - client().prepareIndex("test2").setId("2").setSource("foo", "baz") + client().prepareIndex(INDEX_1).setId("1").setSource("foo", "bar"), + client().prepareIndex(INDEX_1).setId("2").setSource("foo", "baz"), + client().prepareIndex(INDEX_2).setId("1").setSource("foo", "bar"), + client().prepareIndex(INDEX_2).setId("2").setSource("foo", "baz") ); refresh(); // Search with custom plugin to ensure that queryTime is significant - client().prepareSearch("_all") + client().prepareSearch(INDEX_1, INDEX_2) .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", ScriptedDelayedPlugin.SCRIPT_NAME, Collections.emptyMap()))) .execute() .actionGet(); - client().prepareSearch("test1") + client().prepareSearch(INDEX_1) .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", ScriptedDelayedPlugin.SCRIPT_NAME, Collections.emptyMap()))) .execute() .actionGet(); - client().prepareSearch("test2") + client().prepareSearch(INDEX_2) .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", ScriptedDelayedPlugin.SCRIPT_NAME, Collections.emptyMap()))) .execute() .actionGet(); @@ -127,6 +136,177 @@ public void testConcurrentQueryCount() throws Exception { ); } + /** + * Test average concurrency is correctly calculated across indices for the same node + */ + public void testAvgConcurrencyNodeLevel() throws InterruptedException { + int NUM_SHARDS = 1; + String INDEX_1 = "test-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + String INDEX_2 = "test-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + + // Create index test1 with 4 segments + createIndex( + INDEX_1, + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build() + ); + ensureGreen(); + for (int i = 0; i < 4; i++) { + client().prepareIndex(INDEX_1).setId(Integer.toString(i)).setSource("field", "value" + i).get(); + refresh(); + } + + client().prepareSearch(INDEX_1).execute().actionGet(); + NodesStatsResponse nodesStatsResponse = client().admin().cluster().prepareNodesStats().execute().actionGet(); + + assertEquals(1, nodesStatsResponse.getNodes().size(), 0); + double expectedConcurrency = SEGMENT_SLICE_COUNT; + assertEquals( + SEGMENT_SLICE_COUNT, + nodesStatsResponse.getNodes().get(0).getIndices().getSearch().getTotal().getConcurrentAvgSliceCount(), + 0 + ); + + forceMerge(); + // Sleep to make sure force merge completes + Thread.sleep(1000); + client().prepareSearch(INDEX_1).execute().actionGet(); + + nodesStatsResponse = client().admin().cluster().prepareNodesStats().execute().actionGet(); + + assertEquals(1, nodesStatsResponse.getNodes().size(), 0); + expectedConcurrency = (SEGMENT_SLICE_COUNT + 1) / 2.0; + assertEquals( + expectedConcurrency, + nodesStatsResponse.getNodes().get(0).getIndices().getSearch().getTotal().getConcurrentAvgSliceCount(), + 0 + ); + + // Create second index test2 with 4 segments + createIndex( + INDEX_2, + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build() + ); + ensureGreen(); + for (int i = 0; i < 4; i++) { + client().prepareIndex(INDEX_2).setId(Integer.toString(i)).setSource("field", "value" + i).get(); + refresh(); + } + + client().prepareSearch(INDEX_2).execute().actionGet(); + nodesStatsResponse = client().admin().cluster().prepareNodesStats().execute().actionGet(); + + assertEquals(1, nodesStatsResponse.getNodes().size(), 0); + expectedConcurrency = (SEGMENT_SLICE_COUNT + 1 + SEGMENT_SLICE_COUNT) / 3.0; + assertEquals( + expectedConcurrency, + nodesStatsResponse.getNodes().get(0).getIndices().getSearch().getTotal().getConcurrentAvgSliceCount(), + 0 + ); + + forceMerge(); + // Sleep to make sure force merge completes + Thread.sleep(1000); + client().prepareSearch(INDEX_2).execute().actionGet(); + nodesStatsResponse = client().admin().cluster().prepareNodesStats().execute().actionGet(); + + assertEquals(1, nodesStatsResponse.getNodes().size(), 0); + expectedConcurrency = (SEGMENT_SLICE_COUNT + 1 + SEGMENT_SLICE_COUNT + 1) / 4.0; + assertEquals( + expectedConcurrency, + nodesStatsResponse.getNodes().get(0).getIndices().getSearch().getTotal().getConcurrentAvgSliceCount(), + 0 + ); + + // Check that non-concurrent search requests do not affect the average concurrency + client().admin() + .indices() + .prepareUpdateSettings(INDEX_1) + .setSettings(Settings.builder().put("search.concurrent_segment_search.enabled", false)) + .execute() + .actionGet(); + client().admin() + .indices() + .prepareUpdateSettings(INDEX_2) + .setSettings(Settings.builder().put("search.concurrent_segment_search.enabled", false)) + .execute() + .actionGet(); + client().prepareSearch(INDEX_1).execute().actionGet(); + client().prepareSearch(INDEX_2).execute().actionGet(); + assertEquals(1, nodesStatsResponse.getNodes().size(), 0); + assertEquals( + expectedConcurrency, + nodesStatsResponse.getNodes().get(0).getIndices().getSearch().getTotal().getConcurrentAvgSliceCount(), + 0 + ); + } + + /** + * Test average concurrency is correctly calculated across shard for the same index + */ + public void testAvgConcurrencyIndexLevel() throws InterruptedException { + int NUM_SHARDS = 2; + String INDEX = "test-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + createIndex( + INDEX, + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build() + ); + ensureGreen(); + // Create 4 segments on each shard + for (int i = 0; i < 4; i++) { + client().prepareIndex(INDEX).setId(Integer.toString(i)).setSource("field", "value" + i).setRouting("0").get(); + refresh(); + } + for (int i = 4; i < 8; i++) { + client().prepareIndex(INDEX).setId(Integer.toString(i)).setSource("field", "value" + i).setRouting("1").get(); + refresh(); + } + client().prepareSearch(INDEX).execute().actionGet(); + IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats().execute().actionGet(); + + IndexStats stats = indicesStatsResponse.getIndices().get(INDEX); + assertNotNull(stats); + double expectedConcurrency = (SEGMENT_SLICE_COUNT * NUM_SHARDS) / (double) NUM_SHARDS; + assertEquals(expectedConcurrency, stats.getTotal().getSearch().getTotal().getConcurrentAvgSliceCount(), 0); + + forceMerge(); + // Sleep to make sure force merge completes + Thread.sleep(1000); + client().prepareSearch(INDEX).execute().actionGet(); + + indicesStatsResponse = client().admin().indices().prepareStats().execute().actionGet(); + stats = indicesStatsResponse.getIndices().get(INDEX); + assertNotNull(stats); + expectedConcurrency = (SEGMENT_SLICE_COUNT * NUM_SHARDS + 1 * NUM_SHARDS) / (NUM_SHARDS * 2.0); + assertEquals(expectedConcurrency, stats.getTotal().getSearch().getTotal().getConcurrentAvgSliceCount(), 0); + + // Check that non-concurrent search requests do not affect the average concurrency + client().admin() + .indices() + .prepareUpdateSettings(INDEX) + .setSettings(Settings.builder().put("search.concurrent_segment_search.enabled", false)) + .execute() + .actionGet(); + + client().prepareSearch(INDEX).execute().actionGet(); + + indicesStatsResponse = client().admin().indices().prepareStats().execute().actionGet(); + stats = indicesStatsResponse.getIndices().get(INDEX); + assertNotNull(stats); + assertEquals(expectedConcurrency, stats.getTotal().getSearch().getTotal().getConcurrentAvgSliceCount(), 0); + } + public static class ScriptedDelayedPlugin extends MockScriptPlugin { static final String SCRIPT_NAME = "search_timeout"; diff --git a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java index 26e12114973c3..76b216304b3b7 100644 --- a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java +++ b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java @@ -35,6 +35,7 @@ import org.opensearch.Version; import org.opensearch.common.Nullable; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -70,6 +71,7 @@ public static class Stats implements Writeable, ToXContentFragment { private long concurrentQueryCount; private long concurrentQueryTimeInMillis; private long concurrentQueryCurrent; + private long queryConcurrency; private long fetchCount; private long fetchTimeInMillis; @@ -98,6 +100,7 @@ public Stats( long concurrentQueryCount, long concurrentQueryTimeInMillis, long concurrentQueryCurrent, + long queryConcurrency, long fetchCount, long fetchTimeInMillis, long fetchCurrent, @@ -118,6 +121,7 @@ public Stats( this.concurrentQueryCount = concurrentQueryCount; this.concurrentQueryTimeInMillis = concurrentQueryTimeInMillis; this.concurrentQueryCurrent = concurrentQueryCurrent; + this.queryConcurrency = queryConcurrency; this.fetchCount = fetchCount; this.fetchTimeInMillis = fetchTimeInMillis; @@ -163,6 +167,7 @@ private Stats(StreamInput in) throws IOException { concurrentQueryCount = in.readVLong(); concurrentQueryTimeInMillis = in.readVLong(); concurrentQueryCurrent = in.readVLong(); + queryConcurrency = in.readVLong(); } } @@ -174,6 +179,7 @@ public void add(Stats stats) { concurrentQueryCount += stats.concurrentQueryCount; concurrentQueryTimeInMillis += stats.concurrentQueryTimeInMillis; concurrentQueryCurrent += stats.concurrentQueryCurrent; + queryConcurrency += stats.queryConcurrency; fetchCount += stats.fetchCount; fetchTimeInMillis += stats.fetchTimeInMillis; @@ -213,6 +219,7 @@ public void addForClosingShard(Stats stats) { pitCount += stats.pitCount; pitTimeInMillis += stats.pitTimeInMillis; pitCurrent += stats.pitCurrent; + queryConcurrency += stats.queryConcurrency; } public long getQueryCount() { @@ -239,6 +246,14 @@ public TimeValue getConcurrentQueryTime() { return new TimeValue(concurrentQueryTimeInMillis); } + public double getConcurrentAvgSliceCount() { + if (concurrentQueryCount == 0) { + return 0; + } else { + return queryConcurrency / (double) concurrentQueryCount; + } + } + public long getConcurrentQueryTimeInMillis() { return concurrentQueryTimeInMillis; } @@ -343,6 +358,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(concurrentQueryCount); out.writeVLong(concurrentQueryTimeInMillis); out.writeVLong(concurrentQueryCurrent); + out.writeVLong(queryConcurrency); } } @@ -352,9 +368,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.humanReadableField(Fields.QUERY_TIME_IN_MILLIS, Fields.QUERY_TIME, getQueryTime()); builder.field(Fields.QUERY_CURRENT, queryCurrent); - builder.field(Fields.CONCURRENT_QUERY_TOTAL, concurrentQueryCount); - builder.humanReadableField(Fields.CONCURRENT_QUERY_TIME_IN_MILLIS, Fields.CONCURRENT_QUERY_TIME, getConcurrentQueryTime()); - builder.field(Fields.CONCURRENT_QUERY_CURRENT, concurrentQueryCurrent); + if (FeatureFlags.isEnabled(FeatureFlags.CONCURRENT_SEGMENT_SEARCH)) { + builder.field(Fields.CONCURRENT_QUERY_TOTAL, concurrentQueryCount); + builder.humanReadableField(Fields.CONCURRENT_QUERY_TIME_IN_MILLIS, Fields.CONCURRENT_QUERY_TIME, getConcurrentQueryTime()); + builder.field(Fields.CONCURRENT_QUERY_CURRENT, concurrentQueryCurrent); + builder.field(Fields.CONCURRENT_AVG_SLICE_COUNT, getConcurrentAvgSliceCount()); + } builder.field(Fields.FETCH_TOTAL, fetchCount); builder.humanReadableField(Fields.FETCH_TIME_IN_MILLIS, Fields.FETCH_TIME, getFetchTime()); @@ -484,6 +503,7 @@ static final class Fields { static final String CONCURRENT_QUERY_TIME = "concurrent_query_time"; static final String CONCURRENT_QUERY_TIME_IN_MILLIS = "concurrent_query_time_in_millis"; static final String CONCURRENT_QUERY_CURRENT = "concurrent_query_current"; + static final String CONCURRENT_AVG_SLICE_COUNT = "concurrent_avg_slice_count"; static final String FETCH_TOTAL = "fetch_total"; static final String FETCH_TIME = "fetch_time"; static final String FETCH_TIME_IN_MILLIS = "fetch_time_in_millis"; diff --git a/server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java b/server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java index c0d4669413de0..99e3f8465c5db 100644 --- a/server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java +++ b/server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java @@ -130,6 +130,8 @@ public void onQueryPhase(SearchContext searchContext, long tookInNanos) { statsHolder.concurrentQueryMetric.inc(tookInNanos); statsHolder.concurrentQueryCurrent.dec(); assert statsHolder.concurrentQueryCurrent.count() >= 0; + assert searchContext.searcher().getSlices() != null; + statsHolder.queryConcurrencyMetric.inc(searchContext.searcher().getSlices().length); } } }); @@ -219,6 +221,7 @@ public void onFreePitContext(ReaderContext readerContext) { static final class StatsHolder { final MeanMetric queryMetric = new MeanMetric(); final MeanMetric concurrentQueryMetric = new MeanMetric(); + final CounterMetric queryConcurrencyMetric = new CounterMetric(); final MeanMetric fetchMetric = new MeanMetric(); /* We store scroll statistics in microseconds because with nanoseconds we run the risk of overflowing the total stats if there are * many scrolls. For example, on a system with 2^24 scrolls that have been executed, each executing for 2^10 seconds, then using @@ -245,6 +248,7 @@ SearchStats.Stats stats() { concurrentQueryMetric.count(), TimeUnit.NANOSECONDS.toMillis(concurrentQueryMetric.sum()), concurrentQueryCurrent.count(), + queryConcurrencyMetric.count(), fetchMetric.count(), TimeUnit.NANOSECONDS.toMillis(fetchMetric.sum()), fetchCurrent.count(), diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 3494742c0c27a..dea4f02aef5bd 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -53,6 +53,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.Strings; @@ -574,23 +575,31 @@ protected Table getTableWithHeader(final RestRequest request) { "sibling:pri;alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops" ); table.addCell("pri.search.query_total", "default:false;text-align:right;desc:total query phase ops"); - table.addCell( - "search.concurrent_query_current", - "sibling:pri;alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops" - ); - table.addCell("pri.search.concurrent_query_current", "default:false;text-align:right;desc:current concurrent query phase ops"); + if (FeatureFlags.isEnabled(FeatureFlags.CONCURRENT_SEGMENT_SEARCH)) { + table.addCell( + "search.concurrent_query_current", + "sibling:pri;alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops" + ); + table.addCell("pri.search.concurrent_query_current", "default:false;text-align:right;desc:current concurrent query phase ops"); - table.addCell( - "search.concurrent_query_time", - "sibling:pri;alias:scqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase" - ); - table.addCell("pri.search.concurrent_query_time", "default:false;text-align:right;desc:time spent in concurrent query phase"); + table.addCell( + "search.concurrent_query_time", + "sibling:pri;alias:scqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase" + ); + table.addCell("pri.search.concurrent_query_time", "default:false;text-align:right;desc:time spent in concurrent query phase"); - table.addCell( - "search.concurrent_query_total", - "sibling:pri;alias:scqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total query phase ops" - ); - table.addCell("pri.search.concurrent_query_total", "default:false;text-align:right;desc:total query phase ops"); + table.addCell( + "search.concurrent_query_total", + "sibling:pri;alias:scqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total query phase ops" + ); + table.addCell("pri.search.concurrent_query_total", "default:false;text-align:right;desc:total query phase ops"); + + table.addCell( + "search.concurrent_avg_slice_count", + "sibling:pri;alias:casc,searchConcurrentAvgSliceCount;default:false;text-align:right;desc:average query concurrency" + ); + table.addCell("pri.search.concurrent_avg_slice_count", "default:false;text-align:right;desc:average query concurrency"); + } table.addCell( "search.scroll_current", @@ -900,14 +909,19 @@ Table buildTable( table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCount()); table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCurrent()); + if (FeatureFlags.isEnabled(FeatureFlags.CONCURRENT_SEGMENT_SEARCH)) { + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCurrent()); + + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount()); + } table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent()); table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent()); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java index 85864404b2cb1..bf8b27f378311 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java @@ -47,6 +47,7 @@ import org.opensearch.common.Table; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.network.NetworkAddress; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.common.unit.ByteSizeValue; @@ -303,18 +304,24 @@ protected Table getTableWithHeader(final RestRequest request) { table.addCell("search.query_current", "alias:sqc,searchQueryCurrent;default:false;text-align:right;desc:current query phase ops"); table.addCell("search.query_time", "alias:sqti,searchQueryTime;default:false;text-align:right;desc:time spent in query phase"); table.addCell("search.query_total", "alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops"); - table.addCell( - "search.concurrent_query_current", - "alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops" - ); - table.addCell( - "search.concurrent_query_time", - "alias:scqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase" - ); - table.addCell( - "search.concurrent_query_total", - "alias:scqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total concurrent query phase ops" - ); + if (FeatureFlags.isEnabled(FeatureFlags.CONCURRENT_SEGMENT_SEARCH)) { + table.addCell( + "search.concurrent_query_current", + "alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops" + ); + table.addCell( + "search.concurrent_query_time", + "alias:scqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase" + ); + table.addCell( + "search.concurrent_query_total", + "alias:scqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total concurrent query phase ops" + ); + table.addCell( + "search.concurrent_avg_slice_count", + "alias:casc,searchConcurrentAvgSliceCount;default:false;text-align:right;desc:average query concurrency" + ); + } table.addCell("search.scroll_current", "alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts"); table.addCell( "search.scroll_time", @@ -541,9 +548,12 @@ Table buildTable( table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryCurrent()); table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryTime()); table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryCount()); - table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentQueryCurrent()); - table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentQueryTime()); - table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentQueryCount()); + if (FeatureFlags.isEnabled(FeatureFlags.CONCURRENT_SEGMENT_SEARCH)) { + table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentQueryCurrent()); + table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentQueryTime()); + table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentQueryCount()); + table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentAvgSliceCount()); + } table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollCurrent()); table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollTime()); table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollCount()); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java index c14ca68bc7abd..06b31270f7c65 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java @@ -44,6 +44,7 @@ import org.opensearch.common.Table; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import org.opensearch.index.cache.query.QueryCacheStats; import org.opensearch.index.engine.CommitStats; @@ -219,18 +220,24 @@ protected Table getTableWithHeader(final RestRequest request) { table.addCell("search.query_current", "alias:sqc,searchQueryCurrent;default:false;text-align:right;desc:current query phase ops"); table.addCell("search.query_time", "alias:sqti,searchQueryTime;default:false;text-align:right;desc:time spent in query phase"); table.addCell("search.query_total", "alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops"); - table.addCell( - "search.concurrent_query_current", - "alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops" - ); - table.addCell( - "search.concurrent_query_time", - "alias:scqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase" - ); - table.addCell( - "search.concurrent_query_total", - "alias:scqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total concurrent query phase ops" - ); + if (FeatureFlags.isEnabled(FeatureFlags.CONCURRENT_SEGMENT_SEARCH)) { + table.addCell( + "search.concurrent_query_current", + "alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops" + ); + table.addCell( + "search.concurrent_query_time", + "alias:scqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase" + ); + table.addCell( + "search.concurrent_query_total", + "alias:scqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total concurrent query phase ops" + ); + table.addCell( + "search.concurrent_avg_slice_count", + "alias:casc,searchConcurrentAvgSliceCount;default:false;text-align:right;desc:average query concurrency" + ); + } table.addCell("search.scroll_current", "alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts"); table.addCell( "search.scroll_time", @@ -411,9 +418,13 @@ Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsRe table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCurrent())); table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryTime())); table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCount())); - table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryCurrent())); - table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryTime())); - table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryCount())); + if (FeatureFlags.isEnabled(FeatureFlags.CONCURRENT_SEGMENT_SEARCH)) { + table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryCurrent())); + table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryTime())); + table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryCount())); + table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentAvgSliceCount())); + + } table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCurrent())); table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollTime())); table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCount())); diff --git a/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java b/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java index e56fc28235182..f65b0e231b1dd 100644 --- a/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java +++ b/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java @@ -45,9 +45,9 @@ public void testShardLevelSearchGroupStats() throws Exception { // let's create two dummy search stats with groups Map groupStats1 = new HashMap<>(); Map groupStats2 = new HashMap<>(); - groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)); - SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1); - SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2); + groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)); + SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1); + SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2); // adding these two search stats and checking group stats are correct searchStats1.add(searchStats2); @@ -84,6 +84,8 @@ private static void assertStats(Stats stats, long equalTo) { assertEquals(equalTo, stats.getSuggestCount()); assertEquals(equalTo, stats.getSuggestTimeInMillis()); assertEquals(equalTo, stats.getSuggestCurrent()); + // avg_concurrency is not summed up across stats + assertEquals(1, stats.getConcurrentAvgSliceCount(), 0); } } diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java index 3895a64474cc4..a8679a087216d 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java @@ -134,8 +134,8 @@ public void testBuildTable() { assertThat(row.get(3).value, equalTo(shardRouting.state())); assertThat(row.get(6).value, equalTo(localNode.getHostAddress())); assertThat(row.get(7).value, equalTo(localNode.getId())); - assertThat(row.get(75).value, equalTo(shardStats.getDataPath())); - assertThat(row.get(76).value, equalTo(shardStats.getStatePath())); + assertThat(row.get(72).value, equalTo(shardStats.getDataPath())); + assertThat(row.get(73).value, equalTo(shardStats.getStatePath())); } } }