improve the unit test and integ test coverage for query insight plugin

Signed-off-by: Chenyang Ji <cyji@amazon.com>

plugins/query-insights/build.gradle

@@ -10,7 +10,7 @@
  */
 
 //apply plugin: 'opensearch.java-rest-test'
-//apply plugin: 'opensearch.internal-cluster-test'
+apply plugin: 'opensearch.internal-cluster-test'
 
 import org.opensearch.gradle.testclusters.RunTask
 import org.opensearch.gradle.test.RestIntegTestTask

plugins/query-insights/src/internalClusterTest/java/org/opensearch/plugin/insights/QueryInsightsPluginTransportIT.java

@@ -0,0 +1,182 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.plugin.insights;
+
+import org.junit.Test;
+import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.opensearch.action.admin.cluster.node.info.NodeInfo;
+import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest;
+import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.opensearch.action.admin.cluster.node.info.PluginsAndModules;
+import org.opensearch.action.index.IndexResponse;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesAction;
+import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesRequest;
+import org.opensearch.plugin.insights.rules.action.top_queries.TopQueriesResponse;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.plugins.PluginInfo;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.junit.Assert;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_LATENCY_QUERIES_ENABLED;
+import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE;
+import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_LATENCY_QUERIES_WINDOW_SIZE;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+/**
+ * Transport Action tests for Query Insights Plugin
+ */
+
+@OpenSearchIntegTestCase.ClusterScope(numDataNodes = 0, scope = OpenSearchIntegTestCase.Scope.TEST)
+public class QueryInsightsPluginTransportIT extends OpenSearchIntegTestCase {
+
+    private final int TOTAL_NUMBER_OF_NODES = 2;
+    private final int TOTAL_SEARCH_REQUESTS = 5;
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(QueryInsightsPlugin.class);
+    }
+
+    /**
+     * Test Query Insights Plugin is installed
+     */
+    public void testQueryInsightPluginInstalled() {
+        NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
+        nodesInfoRequest.addMetric(NodesInfoRequest.Metric.PLUGINS.metricName());
+        NodesInfoResponse nodesInfoResponse = OpenSearchIntegTestCase.client().admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
+        List<PluginInfo> pluginInfos = nodesInfoResponse.getNodes()
+            .stream()
+            .flatMap(
+                (Function<NodeInfo, Stream<PluginInfo>>) nodeInfo -> nodeInfo.getInfo(PluginsAndModules.class).getPluginInfos().stream()
+            )
+            .collect(Collectors.toList());
+        Assert.assertTrue(
+            pluginInfos.stream()
+                .anyMatch(pluginInfo -> pluginInfo.getName().equals("org.opensearch.plugin.insights.QueryInsightsPlugin"))
+        );
+    }
+
+    /**
+     * Test get top queries when feature disabled
+     */
+    public void testGetTopQueriesWhenFeatureDisabled() {
+        TopQueriesRequest request = new TopQueriesRequest();
+        TopQueriesResponse response = OpenSearchIntegTestCase.client().execute(TopQueriesAction.INSTANCE, request).actionGet();
+        Assert.assertNotEquals(0, response.failures().size());
+        Assert.assertEquals("Cannot get query data when query insight feature is not enabled.", response.failures().get(0).getCause().getCause().getMessage());
+    }
+
+    /**
+     * Test get top queries when feature enabled
+     */
+    public void testGetTopQueriesWhenFeatureEnabled() {
+        Settings commonSettings = Settings.builder()
+            .put(TOP_N_LATENCY_QUERIES_ENABLED.getKey(), "true")
+            .put(TOP_N_LATENCY_QUERIES_SIZE.getKey(), "100")
+            .put(TOP_N_LATENCY_QUERIES_WINDOW_SIZE.getKey(), "600s")
+            .build();
+
+        logger.info("--> starting 2 nodes for query insight testing");
+        List<String> nodes = internalCluster().startNodes(
+            TOTAL_NUMBER_OF_NODES,
+            Settings.builder().put(commonSettings).build()
+        );
+
+        logger.info("--> waiting for nodes to form a cluster");
+        ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet();
+        assertFalse(health.isTimedOut());
+
+        assertAcked(
+            prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 2))
+        );
+        ensureStableCluster(2);
+        logger.info("--> creating indices for query insight testing");
+        for (int i = 0; i < 5; i++) {
+            IndexResponse response = client().prepareIndex("test_" + i).setId("" + i).setSource("field_" + i, "value_" + i).get();
+            assertEquals("CREATED", response.status().toString());
+        }
+        // making search requests to get top queries
+        for (int i = 0; i < TOTAL_SEARCH_REQUESTS; i++) {
+            SearchResponse searchResponse = internalCluster().client(randomFrom(nodes))
+                .prepareSearch()
+                .setQuery(QueryBuilders.matchAllQuery())
+                .get();
+            assertEquals(searchResponse.getFailedShards(), 0);
+        }
+
+        TopQueriesRequest request = new TopQueriesRequest();
+        TopQueriesResponse response = OpenSearchIntegTestCase.client().execute(TopQueriesAction.INSTANCE, request).actionGet();
+        Assert.assertEquals(0, response.failures().size());
+        Assert.assertEquals(TOTAL_NUMBER_OF_NODES, response.getNodes().size());
+        Assert.assertEquals(TOTAL_SEARCH_REQUESTS, response.getNodes().stream().mapToInt(o -> o.getLatencyRecords().size()).sum());
+
+        internalCluster().stopAllNodes();
+    }
+
+    /**
+     * Test get top queries with small top n size
+     */
+
+    @Test
+    public void testGetTopQueriesWithSmallTopN() {
+        Settings commonSettings = Settings.builder()
+            .put(TOP_N_LATENCY_QUERIES_ENABLED.getKey(), "true")
+            .put(TOP_N_LATENCY_QUERIES_SIZE.getKey(), "1")
+            .put(TOP_N_LATENCY_QUERIES_WINDOW_SIZE.getKey(), "600s")
+            .build();
+
+        logger.info("--> starting 2 nodes for query insight testing");
+        List<String> nodes = internalCluster().startNodes(
+            TOTAL_NUMBER_OF_NODES,
+            Settings.builder().put(commonSettings).build()
+        );
+
+        logger.info("--> waiting for nodes to form a cluster");
+        ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet();
+        assertFalse(health.isTimedOut());
+
+        assertAcked(
+            prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 2))
+        );
+        ensureStableCluster(2);
+        logger.info("--> creating indices for query insight testing");
+        for (int i = 0; i < 5; i++) {
+            IndexResponse response = client().prepareIndex("test_" + i).setId("" + i).setSource("field_" + i, "value_" + i).get();
+            assertEquals("CREATED", response.status().toString());
+        }
+        // making search requests to get top queries
+        for (int i = 0; i < TOTAL_SEARCH_REQUESTS; i++) {
+            SearchResponse searchResponse = internalCluster().client(randomFrom(nodes))
+                .prepareSearch()
+                .setQuery(QueryBuilders.matchAllQuery())
+                .get();
+            assertEquals(searchResponse.getFailedShards(), 0);
+        }
+
+
+        TopQueriesRequest request = new TopQueriesRequest();
+        TopQueriesResponse response = OpenSearchIntegTestCase.client().execute(TopQueriesAction.INSTANCE, request).actionGet();
+        Assert.assertEquals(0, response.failures().size());
+        Assert.assertEquals(TOTAL_NUMBER_OF_NODES, response.getNodes().size());
+        // TODO: this should be 1 after changing to cluster level top N.
+        Assert.assertEquals(2, response.getNodes().stream().mapToInt(o -> o.getLatencyRecords().size()).sum());
+
+        internalCluster().stopAllNodes();
+    }
+}

plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/listener/SearchQueryLatencyListener.java

@@ -42,6 +42,9 @@ public final class SearchQueryLatencyListener extends SearchRequestOperationsLis
     @Inject
     public SearchQueryLatencyListener(ClusterService clusterService, TopQueriesByLatencyService topQueriesByLatencyService) {
         this.topQueriesByLatencyService = topQueriesByLatencyService;
+        this.setEnabled(clusterService.getClusterSettings().get(TOP_N_LATENCY_QUERIES_ENABLED));
+        this.topQueriesByLatencyService.setTopNSize(clusterService.getClusterSettings().get(TOP_N_LATENCY_QUERIES_SIZE));
+        this.topQueriesByLatencyService.setWindowSize(clusterService.getClusterSettings().get(TOP_N_LATENCY_QUERIES_WINDOW_SIZE));
         clusterService.getClusterSettings().addSettingsUpdateConsumer(TOP_N_LATENCY_QUERIES_ENABLED, this::setEnabled);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(TOP_N_LATENCY_QUERIES_SIZE, this.topQueriesByLatencyService::setTopNSize);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(TOP_N_LATENCY_QUERIES_WINDOW_SIZE, this.topQueriesByLatencyService::setWindowSize);

plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponse.java

@@ -17,6 +17,7 @@
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.plugin.insights.core.service.TopQueriesByLatencyService;
 import org.opensearch.plugin.insights.rules.model.SearchQueryLatencyRecord;
 
 import java.io.IOException;
@@ -82,6 +83,7 @@ public String toString() {
      * @param results top queries results from all nodes
      * @throws IOException if an error occurs
      */
+    // TODO: how to get cluster level "top n" from node level top n?
     private void toClusterLevelResult(XContentBuilder builder, Params params, List<TopQueries> results) throws IOException {
         List<SearchQueryLatencyRecord> all_records = results.stream()
             .map(TopQueries::getLatencyRecords)

plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryLatencyRecord.java

@@ -69,4 +69,19 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeMap(phaseLatencyMap, StreamOutput::writeString, StreamOutput::writeLong);
     }
+
+    public boolean equals(SearchQueryLatencyRecord other) {
+        if (!super.equals(other)) {
+            return false;
+        }
+        for (String key : phaseLatencyMap.keySet()) {
+            if (!other.getPhaseLatencyMap().containsKey(key)) {
+                return false;
+            }
+            if(!phaseLatencyMap.get(key).equals(other.getPhaseLatencyMap().get(key))) {
+                return false;
+            }
+        }
+        return true;
+    }
 }

plugins/query-insights/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java

@@ -158,6 +158,32 @@ public int compareTo(SearchQueryRecord<T> otherRecord) {
         return value.compareTo(otherRecord.getValue());
     }
 
+    public boolean equals(SearchQueryRecord<T> other) {
+        if(false == this.timestamp.equals(other.getTimestamp()) &&
+            this.searchType.equals(other.getSearchType()) &&
+            this.source.equals(other.getSource()) &&
+            this.totalShards == other.getTotalShards() &&
+            this.indices.length == other.getIndices().length &&
+            this.propertyMap.size() == other.getPropertyMap().size() &&
+            this.value.equals(other.getValue())) {
+            return false;
+        }
+        for (int i = 0; i < indices.length; i++) {
+            if (!indices[i].equals(other.getIndices()[i])) {
+                return false;
+            }
+        }
+        for (String key : propertyMap.keySet()) {
+            if (!other.getPropertyMap().containsKey(key)) {
+                return false;
+            }
+            if(!propertyMap.get(key).equals(other.getPropertyMap().get(key))) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     @SuppressWarnings("unchecked")
     private T castToValue(Object obj) throws ClassCastException {
         try {