Search latency tracking - Coordinator node

server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java

@@ -57,7 +57,9 @@
 import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.opensearch.action.search.TransportSearchAction.SEARCH_REQUEST_STATS_ENABLED_KEY;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.search.aggregations.AggregationBuilders.terms;
 
@@ -74,6 +76,7 @@ public void testSearchWithWRRShardRouting() throws IOException {
             .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone" + ".values", "a,b,c")
             .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone")
             .put("cluster.routing.weighted.fail_open", false)
+            .put(SEARCH_REQUEST_STATS_ENABLED_KEY, true)
             .build();
 
         logger.info("--> starting 6 nodes on different zones");
@@ -180,12 +183,24 @@ public void testSearchWithWRRShardRouting() throws IOException {
             assertFalse(!hitNodes.contains(nodeId));
         }
         nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet();
+        int num = 0;
+        int coordNumber = 0;
 
         for (NodeStats stat : nodeStats.getNodes()) {
             SearchStats.Stats searchStats = stat.getIndices().getSearch().getTotal();
+            if (searchStats.getRequestStatsLongHolder().queryMetric > 0) {
+                assertThat(searchStats.getRequestStatsLongHolder().queryTotal, greaterThan(0L));
+                assertThat(searchStats.getRequestStatsLongHolder().fetchMetric, greaterThan(0L));
+                assertThat(searchStats.getRequestStatsLongHolder().fetchTotal, greaterThan(0L));
+                assertThat(searchStats.getRequestStatsLongHolder().expandSearchTotal, greaterThan(0L));
+                coordNumber += 1;
+            }
             Assert.assertTrue(searchStats.getQueryCount() > 0L);
             Assert.assertTrue(searchStats.getFetchCount() > 0L);
+            num++;
         }
+        assertThat(coordNumber, greaterThan(0));
+        assertThat(num, greaterThan(0));
     }
 
     private Map<String, List<String>> setupCluster(int nodeCountPerAZ, Settings commonSettings) {

server/src/internalClusterTest/java/org/opensearch/search/stats/SearchStatsIT.java

@@ -58,6 +58,7 @@
 import java.util.Set;
 import java.util.function.Function;
 
+import static org.opensearch.action.search.TransportSearchAction.SEARCH_REQUEST_STATS_ENABLED_KEY;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
 import static org.opensearch.index.query.QueryBuilders.matchAllQuery;
@@ -72,7 +73,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-@OpenSearchIntegTestCase.ClusterScope(minNumDataNodes = 2)
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, minNumDataNodes = 2)
 public class SearchStatsIT extends OpenSearchIntegTestCase {
 
     @Override
@@ -103,6 +104,11 @@ public void testSimpleStats() throws Exception {
         assertThat(numNodes, greaterThanOrEqualTo(2));
         final int shardsIdx1 = randomIntBetween(1, 10); // we make sure each node gets at least a single shard...
         final int shardsIdx2 = Math.max(numNodes - shardsIdx1, randomIntBetween(1, 10));
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setPersistentSettings(Settings.builder().put(SEARCH_REQUEST_STATS_ENABLED_KEY, true).build())
+            .get();
         assertThat(numNodes, lessThanOrEqualTo(shardsIdx1 + shardsIdx2));
         assertAcked(
             prepareCreate("test1").setSettings(
@@ -165,20 +171,28 @@ public void testSimpleStats() throws Exception {
 
         Set<String> nodeIdsWithIndex = nodeIdsWithIndex("test1", "test2");
         int num = 0;
+        int numOfCoordinators = 0;
+
         for (NodeStats stat : nodeStats.getNodes()) {
             Stats total = stat.getIndices().getSearch().getTotal();
+            if (total.getRequestStatsLongHolder().queryMetric > 0) {
+                assertEquals(total.getRequestStatsLongHolder().queryTotal, iters);
+                assertThat(total.getRequestStatsLongHolder().fetchMetric, greaterThan(0L));
+                assertEquals(total.getRequestStatsLongHolder().fetchTotal, iters);
+                assertEquals(total.getRequestStatsLongHolder().expandSearchTotal, iters);
+                numOfCoordinators += 1;
+            }
             if (nodeIdsWithIndex.contains(stat.getNode().getId())) {
                 assertThat(total.getQueryCount(), greaterThan(0L));
                 assertThat(total.getQueryTimeInMillis(), greaterThan(0L));
                 num++;
             } else {
-                assertThat(total.getQueryCount(), equalTo(0L));
+                assertThat(total.getQueryCount(), greaterThanOrEqualTo(0L));
                 assertThat(total.getQueryTimeInMillis(), equalTo(0L));
             }
         }
-
+        assertThat(numOfCoordinators, greaterThan(0));
         assertThat(num, greaterThan(0));
-
     }
 
     private Set<String> nodeIdsWithIndex(String... indices) {

server/src/main/java/org/opensearch/action/RequestStats.java

@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action;
+
+import org.opensearch.action.search.SearchRequestStats;
+
+/**
+ * Request level stats
+ *
+ * @opensearch.internal
+ */
+public final class RequestStats {
+
+    public SearchRequestStats searchRequestStats;
+
+    public RequestStats() {
+        searchRequestStats = new SearchRequestStats();
+    }
+
+    public SearchRequestStats getSearchRequestStats() {
+        return searchRequestStats;
+    }
+
+}

server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java

@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.indices.stats;
 
 import org.apache.lucene.store.AlreadyClosedException;
+import org.opensearch.action.RequestStats;
 import org.opensearch.common.Nullable;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -284,6 +285,14 @@
         out.writeOptionalWriteable(recoveryStats);
     }
 
+    // We are adding request stats with a separate setter method since SearchStats was tightly coupled with Shard Search Stats, and all
+    // nodes won't share the same response in requestStats
+    public void addRequestStats(RequestStats requestStats) {
+        if (requestStats.getSearchRequestStats() != null && this.search != null) {
+            search.setSearchRequestStats(requestStats.getSearchRequestStats());
+        }
+    }
+
     public void add(CommonStats stats) {
         if (docs == null) {
             if (stats.getDocs() != null) {