Merge branch 'wait_until' of https://github.com/Rishikesh1159/OpenSearch

 into wait_until

.github/workflows/dependabot_pr.yml

@@ -49,7 +49,7 @@ jobs:
           commit_options: '--signoff'
 
       - name: Update the changelog
-        uses: dangoslen/dependabot-changelog-helper@v2
+        uses: dangoslen/dependabot-changelog-helper@v3
         with:
           version: 'Unreleased 3.0'
 

CHANGELOG.md

@@ -34,6 +34,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `joni` from 2.1.44 to 2.1.45
 - Bump `commons-io:commons-io` from 2.7 to 2.11.0
 - Bump `org.jruby.joni:joni` from 2.1.45 to 2.1.48
+- Bump `org.apache.logging.log4j:log4j-core` from 2.19.0 to 2.20.0
+- Bump `com.azure:azure-storage-common` from 12.19.3 to 12.20.0
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -64,12 +66,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
 - Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460))
+- Added depth check in doc parser for deep nested document ([#5199](https://github.com/opensearch-project/OpenSearch/pull/5199))
 
 ### Security
 
 ## [Unreleased 2.x]
 ### Added
 - Add GeoTile and GeoHash Grid aggregations on GeoShapes. ([#5589](https://github.com/opensearch-project/OpenSearch/pull/5589))
+- Disallow multiple data paths for search nodes ([#6427](https://github.com/opensearch-project/OpenSearch/pull/6427))
 
 ### Dependencies
 
@@ -86,4 +90,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.5...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.5...2.x

README.md

@@ -24,7 +24,7 @@
 
 ## Welcome!
 
-**OpenSearch** is [a community-driven, open source fork](https://aws.amazon.com/blogs/opensource/introducing-opensearch/) of [Elasticsearch](https://en.wikipedia.org/wiki/Elasticsearch) and [Kibana](https://en.wikipedia.org/wiki/Kibana) following the [licence change](https://opensource.org/node/1099) in early 2021. We're looking to sustain (and evolve!) a search and analytics suite for the multitude of businesses who are dependent on the rights granted by the original, [Apache v2.0 License](LICENSE.txt).
+**OpenSearch** is [a community-driven, open source fork](https://aws.amazon.com/blogs/opensource/introducing-opensearch/) of [Elasticsearch](https://en.wikipedia.org/wiki/Elasticsearch) and [Kibana](https://en.wikipedia.org/wiki/Kibana) following the [license change](https://blog.opensource.org/the-sspl-is-not-an-open-source-license/) in early 2021. We're looking to sustain (and evolve!) a search and analytics suite for the multitude of businesses who are dependent on the rights granted by the original, [Apache v2.0 License](LICENSE.txt).
 
 ## Project Resources
 

buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle

@@ -16,7 +16,7 @@ repositories {
   mavenCentral()
 }
 dependencies {
-  implementation 'org.apache.logging.log4j:log4j-core:2.19.0'
+  implementation 'org.apache.logging.log4j:log4j-core:2.20.0'
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->

plugins/repository-azure/build.gradle

@@ -45,7 +45,7 @@ opensearchplugin {
 
 dependencies {
   api 'com.azure:azure-core:1.34.0'
-  api 'com.azure:azure-storage-common:12.19.3'
+  api 'com.azure:azure-storage-common:12.20.0'
   api 'com.azure:azure-core-http-netty:1.12.8'
   api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"

plugins/repository-azure/licenses/azure-storage-common-12.20.0.jar.sha1

@@ -0,0 +1 @@
+69b962bbeea787c6aca83115472791aacc2ae94c

server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java

@@ -419,17 +419,6 @@ public void testNewlyAddedReplicaIsUpdated() throws Exception {
         assertAcked(
             client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 1))
         );
-
-        ClusterHealthResponse clusterHealthResponse = client().admin()
-            .cluster()
-            .prepareHealth()
-            .setWaitForEvents(Priority.LANGUID)
-            .setWaitForNodes("2")
-            .setWaitForGreenStatus()
-            .setTimeout(TimeValue.timeValueSeconds(2))
-            .execute()
-            .actionGet();
-        assertFalse(clusterHealthResponse.isTimedOut());
         ensureGreen(INDEX_NAME);
         flushAndRefresh(INDEX_NAME);
         waitForSearchableDocs(20, primary, replica);

server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java

@@ -7,6 +7,8 @@
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.hamcrest.MatcherAssert;
+import org.opensearch.action.admin.cluster.node.stats.NodeStats;
+import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
@@ -29,6 +31,7 @@
 import org.opensearch.index.Index;
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter;
+import org.opensearch.index.store.remote.filecache.FileCacheStats;
 import org.opensearch.monitor.fs.FsInfo;
 import org.opensearch.repositories.fs.FsRepository;
 
@@ -413,4 +416,77 @@ private void assertIndexDirectoryDoesNotExist(String... indexNames) {
             }
         }
     }
+
+    public void testFileCacheStats() throws Exception {
+        final String snapshotName = "test-snap";
+        final String repoName = "test-repo";
+        final String indexName1 = "test-idx-1";
+        final Client client = client();
+        final int numNodes = 2;
+
+        internalCluster().ensureAtLeastNumDataNodes(numNodes);
+        createIndexWithDocsAndEnsureGreen(1, 100, indexName1);
+
+        createRepositoryWithSettings(null, repoName);
+        takeSnapshot(client, snapshotName, repoName, indexName1);
+        deleteIndicesAndEnsureGreen(client, indexName1);
+        assertAllNodesFileCacheEmpty();
+
+        internalCluster().ensureAtLeastNumSearchNodes(numNodes);
+        restoreSnapshotAndEnsureGreen(client, snapshotName, repoName);
+        assertNodesFileCacheNonEmpty(numNodes);
+    }
+
+    private void assertAllNodesFileCacheEmpty() {
+        NodesStatsResponse response = client().admin().cluster().nodesStats(new NodesStatsRequest().all()).actionGet();
+        for (NodeStats stats : response.getNodes()) {
+            FileCacheStats fcstats = stats.getFileCacheStats();
+            if (fcstats != null) {
+                assertTrue(isFileCacheEmpty(fcstats));
+            }
+        }
+    }
+
+    private void assertNodesFileCacheNonEmpty(int numNodes) {
+        NodesStatsResponse response = client().admin().cluster().nodesStats(new NodesStatsRequest().all()).actionGet();
+        int nonEmptyFileCacheNodes = 0;
+        for (NodeStats stats : response.getNodes()) {
+            FileCacheStats fcStats = stats.getFileCacheStats();
+            if (stats.getNode().isSearchNode()) {
+                if (!isFileCacheEmpty(fcStats)) {
+                    nonEmptyFileCacheNodes++;
+                }
+            } else {
+                assertNull(fcStats);
+            }
+
+        }
+        assertEquals(numNodes, nonEmptyFileCacheNodes);
+    }
+
+    private boolean isFileCacheEmpty(FileCacheStats stats) {
+        return stats.getUsed().getBytes() == 0L && stats.getActive().getBytes() == 0L;
+    }
+
+    public void testPruneFileCacheOnIndexDeletion() throws Exception {
+        final String snapshotName = "test-snap";
+        final String repoName = "test-repo";
+        final String indexName1 = "test-idx-1";
+        final String restoredIndexName1 = indexName1 + "-copy";
+        final Client client = client();
+        final int numNodes = 2;
+
+        internalCluster().ensureAtLeastNumSearchNodes(numNodes);
+        createIndexWithDocsAndEnsureGreen(1, 100, indexName1);
+
+        createRepositoryWithSettings(null, repoName);
+        takeSnapshot(client, snapshotName, repoName, indexName1);
+        deleteIndicesAndEnsureGreen(client, indexName1);
+
+        restoreSnapshotAndEnsureGreen(client, snapshotName, repoName);
+        assertNodesFileCacheNonEmpty(numNodes);
+
+        deleteIndicesAndEnsureGreen(client, restoredIndexName1);
+        assertAllNodesFileCacheEmpty();
+    }
 }

server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java

@@ -41,12 +41,14 @@
 import org.opensearch.common.Nullable;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.discovery.DiscoveryStats;
 import org.opensearch.http.HttpStats;
 import org.opensearch.index.stats.IndexingPressureStats;
 import org.opensearch.index.stats.ShardIndexingPressureStats;
+import org.opensearch.index.store.remote.filecache.FileCacheStats;
 import org.opensearch.indices.NodeIndicesStats;
 import org.opensearch.indices.breaker.AllCircuitBreakerStats;
 import org.opensearch.ingest.IngestStats;
@@ -130,6 +132,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
     @Nullable
     private WeightedRoutingStats weightedRoutingStats;
 
+    @Nullable
+    private FileCacheStats fileCacheStats;
+
     public NodeStats(StreamInput in) throws IOException {
         super(in);
         timestamp = in.readVLong();
@@ -171,6 +176,11 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             weightedRoutingStats = null;
         }
+        if (in.getVersion().onOrAfter(Version.V_3_0_0) && FeatureFlags.isEnabled(FeatureFlags.SEARCHABLE_SNAPSHOT)) {
+            fileCacheStats = in.readOptionalWriteable(FileCacheStats::new);
+        } else {
+            fileCacheStats = null;
+        }
     }
 
     public NodeStats(
@@ -194,7 +204,8 @@ public NodeStats(
         @Nullable ShardIndexingPressureStats shardIndexingPressureStats,
         @Nullable SearchBackpressureStats searchBackpressureStats,
         @Nullable ClusterManagerThrottlingStats clusterManagerThrottlingStats,
-        @Nullable WeightedRoutingStats weightedRoutingStats
+        @Nullable WeightedRoutingStats weightedRoutingStats,
+        @Nullable FileCacheStats fileCacheStats
     ) {
         super(node);
         this.timestamp = timestamp;
@@ -217,6 +228,7 @@ public NodeStats(
         this.searchBackpressureStats = searchBackpressureStats;
         this.clusterManagerThrottlingStats = clusterManagerThrottlingStats;
         this.weightedRoutingStats = weightedRoutingStats;
+        this.fileCacheStats = fileCacheStats;
     }
 
     public long getTimestamp() {
@@ -340,6 +352,10 @@ public WeightedRoutingStats getWeightedRoutingStats() {
         return weightedRoutingStats;
     }
 
+    public FileCacheStats getFileCacheStats() {
+        return fileCacheStats;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -374,6 +390,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_6_0)) {
             out.writeOptionalWriteable(weightedRoutingStats);
         }
+        if (out.getVersion().onOrAfter(Version.V_3_0_0) && FeatureFlags.isEnabled(FeatureFlags.SEARCHABLE_SNAPSHOT)) {
+            out.writeOptionalWriteable(fileCacheStats);
+        }
     }
 
     @Override
@@ -455,6 +474,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (getWeightedRoutingStats() != null) {
             getWeightedRoutingStats().toXContent(builder, params);
         }
+        if (getFileCacheStats() != null && FeatureFlags.isEnabled(FeatureFlags.SEARCHABLE_SNAPSHOT)) {
+            getFileCacheStats().toXContent(builder, params);
+        }
 
         return builder;
     }

server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java

@@ -209,7 +209,8 @@ public enum Metric {
         SHARD_INDEXING_PRESSURE("shard_indexing_pressure"),
         SEARCH_BACKPRESSURE("search_backpressure"),
         CLUSTER_MANAGER_THROTTLING("cluster_manager_throttling"),
-        WEIGHTED_ROUTING_STATS("weighted_routing");
+        WEIGHTED_ROUTING_STATS("weighted_routing"),
+        FILE_CACHE_STATS("file_cache");
 
         private String metricName;
 

server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java

@@ -121,7 +121,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
             NodesStatsRequest.Metric.SHARD_INDEXING_PRESSURE.containedIn(metrics),
             NodesStatsRequest.Metric.SEARCH_BACKPRESSURE.containedIn(metrics),
             NodesStatsRequest.Metric.CLUSTER_MANAGER_THROTTLING.containedIn(metrics),
-            NodesStatsRequest.Metric.WEIGHTED_ROUTING_STATS.containedIn(metrics)
+            NodesStatsRequest.Metric.WEIGHTED_ROUTING_STATS.containedIn(metrics),
+            NodesStatsRequest.Metric.FILE_CACHE_STATS.containedIn(metrics)
         );
     }
 

server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java

@@ -165,6 +165,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
             false,
             false,
             false,
+            false,
             false
         );
         List<ShardStats> shardsStats = new ArrayList<>();

server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java

@@ -38,15 +38,18 @@
 import org.apache.lucene.util.Constants;
 import org.opensearch.bootstrap.jvm.DenyJvmVersionsParser;
 import org.opensearch.cluster.coordination.ClusterBootstrapService;
+import org.opensearch.cluster.node.DiscoveryNodeRole;
 import org.opensearch.common.SuppressForbidden;
 import org.opensearch.common.io.PathUtils;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.transport.BoundTransportAddress;
 import org.opensearch.common.transport.TransportAddress;
 import org.opensearch.discovery.DiscoveryModule;
+import org.opensearch.env.Environment;
 import org.opensearch.index.IndexModule;
 import org.opensearch.monitor.jvm.JvmInfo;
 import org.opensearch.monitor.process.ProcessProbe;
+import org.opensearch.node.NodeRoleSettings;
 import org.opensearch.node.NodeValidationException;
 
 import java.io.BufferedReader;
@@ -228,6 +231,7 @@ static List<BootstrapCheck> checks() {
         checks.add(new JavaVersionCheck());
         checks.add(new AllPermissionCheck());
         checks.add(new DiscoveryConfiguredCheck());
+        checks.add(new MultipleDataPathCheck());
         return Collections.unmodifiableList(checks);
     }
 
@@ -751,4 +755,25 @@ public BootstrapCheckResult check(BootstrapContext context) {
             );
         }
     }
+
+    /**
+     * Bootstrap check that if a search node contains multiple data paths
+     */
+    static class MultipleDataPathCheck implements BootstrapCheck {
+
+        @Override
+        public BootstrapCheckResult check(BootstrapContext context) {
+            if (NodeRoleSettings.NODE_ROLES_SETTING.get(context.settings()).contains(DiscoveryNodeRole.SEARCH_ROLE)
+                && Environment.PATH_DATA_SETTING.get(context.settings()).size() > 1) {
+                return BootstrapCheckResult.failure("Multiple data paths are not allowed for search nodes");
+            }
+            return BootstrapCheckResult.success();
+        }
+
+        @Override
+        public final boolean alwaysEnforce() {
+            return true;
+        }
+
+    }
 }

server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java

@@ -121,6 +121,10 @@ public static boolean isRemoteClusterClient(final Settings settings) {
         return hasRole(settings, DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE);
     }
 
+    public static boolean isSearchNode(Settings settings) {
+        return hasRole(settings, DiscoveryNodeRole.SEARCH_ROLE);
+    }
+
     private final String nodeName;
     private final String nodeId;
     private final String ephemeralId;

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

@@ -35,6 +35,7 @@
 import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance;
 import org.opensearch.action.search.CreatePitController;
 import org.opensearch.cluster.routing.allocation.decider.NodeLoadAwareAllocationDecider;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.IndexModule;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.IndexingPressure;
@@ -152,6 +153,7 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Predicate;
 
@@ -629,4 +631,14 @@ public void apply(Settings value, Settings current, Settings previous) {
 
     public static List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList();
 
+    /**
+     * Map of feature flag name to feature-flagged cluster settings. Once each feature
+     * is ready for production release, the feature flag can be removed, and the
+     * setting should be moved to {@link #BUILT_IN_CLUSTER_SETTINGS}.
+     */
+    public static final Map<String, List<Setting>> FEATURE_FLAGGED_CLUSTER_SETTINGS = Map.of(
+        FeatureFlags.SEARCHABLE_SNAPSHOT,
+        List.of(Node.NODE_SEARCH_CACHE_SIZE_SETTING)
+    );
+
 }