From 496f943f468f2d7eedd8cfd3f837c028e65e6e3f Mon Sep 17 00:00:00 2001 From: bansvaru Date: Fri, 21 Jul 2023 11:41:45 +0530 Subject: [PATCH 1/9] Retrict remote only indices in a remote enabled cluster Signed-off-by: bansvaru --- ...CreateRemoteIndexClusterDefaultDocRep.java | 14 +- .../remotestore/CreateRemoteIndexIT.java | 122 +++++++----------- .../metadata/MetadataCreateIndexService.java | 41 +----- .../MetadataCreateIndexServiceTests.java | 120 ++++++++--------- 4 files changed, 117 insertions(+), 180 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java index 2abf4fc50ec69..fbe821d049a17 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java @@ -16,7 +16,11 @@ import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchIntegTestCase; +import java.util.Locale; + import static org.hamcrest.Matchers.containsString; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -44,7 +48,15 @@ public void testDefaultRemoteStoreNoUserOverride() throws Exception { ); assertThat( exc.getMessage(), - containsString("Cannot enable [index.remote_store.enabled] when [index.replication.type] is DOCUMENT") + containsString( + String.format( + Locale.ROOT, + "To enable %s, %s should be set to %s", + SETTING_REMOTE_STORE_ENABLED, + SETTING_REPLICATION_TYPE, + ReplicationType.SEGMENT + ) + ) ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java index e52a12f66cff4..2d2533500bf9d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java @@ -26,14 +26,13 @@ import static org.hamcrest.Matchers.containsString; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; -import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; +import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_SETTING_REPLICATION_TYPE; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) @@ -111,20 +110,12 @@ public void testRemoteStoreDisabledByUser() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(SETTING_REMOTE_STORE_ENABLED, false) .build(); - assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); - GetIndexResponse getIndexResponse = client().admin() - .indices() - .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true)) - .get(); - Settings indexSettings = getIndexResponse.settings().get("test-idx-1"); - verifyRemoteStoreIndexSettings( - indexSettings, - "false", - null, - null, - client().settings().get(CLUSTER_SETTING_REPLICATION_TYPE), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL + + IllegalArgumentException exc = expectThrows( + IllegalArgumentException.class, + () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() ); + assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); } public void testRemoteStoreEnabledByUserWithoutRemoteRepoAndSegmentReplicationIllegalArgumentException() throws Exception { @@ -156,16 +147,7 @@ public void testRemoteStoreEnabledByUserWithoutRemoteRepoIllegalArgumentExceptio IllegalArgumentException.class, () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() ); - assertThat( - exc.getMessage(), - containsString( - String.format( - Locale.ROOT, - "Setting %s should be provided with non-empty repository ID", - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY - ) - ) - ); + assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); } public void testReplicationTypeDocumentByUser() throws Exception { @@ -174,19 +156,21 @@ public void testReplicationTypeDocumentByUser() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) .build(); - assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); - GetIndexResponse getIndexResponse = client().admin() - .indices() - .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true)) - .get(); - Settings indexSettings = getIndexResponse.settings().get("test-idx-1"); - verifyRemoteStoreIndexSettings( - indexSettings, - null, - null, - null, - ReplicationType.DOCUMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL + IllegalArgumentException exc = expectThrows( + IllegalArgumentException.class, + () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() + ); + assertThat( + exc.getMessage(), + containsString( + String.format( + Locale.ROOT, + "To enable %s, %s should be set to %s", + SETTING_REMOTE_STORE_ENABLED, + SETTING_REPLICATION_TYPE, + ReplicationType.SEGMENT + ) + ) ); } @@ -222,20 +206,11 @@ public void testRemoteStoreEnabledByUserWithRemoteRepo() throws Exception { .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo") .build(); - assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); - GetIndexResponse getIndexResponse = client().admin() - .indices() - .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true)) - .get(); - Settings indexSettings = getIndexResponse.settings().get("test-idx-1"); - verifyRemoteStoreIndexSettings( - indexSettings, - "true", - "my-custom-repo", - "my-translog-repo-1", - ReplicationType.SEGMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL + IllegalArgumentException exc = expectThrows( + IllegalArgumentException.class, + () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() ); + assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); } public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() throws Exception { @@ -270,20 +245,11 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception { .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo") .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo") .build(); - assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); - GetIndexResponse getIndexResponse = client().admin() - .indices() - .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true)) - .get(); - Settings indexSettings = getIndexResponse.settings().get("test-idx-1"); - verifyRemoteStoreIndexSettings( - indexSettings, - "true", - "my-custom-repo", - "my-custom-repo", - ReplicationType.SEGMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL + IllegalArgumentException exc = expectThrows( + IllegalArgumentException.class, + () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() ); + assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); } public void testRemoteStoreOverrideReplicationTypeIndexSettings() throws Exception { @@ -292,19 +258,21 @@ public void testRemoteStoreOverrideReplicationTypeIndexSettings() throws Excepti .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) .build(); - assertAcked(client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get()); - GetIndexResponse getIndexResponse = client().admin() - .indices() - .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true)) - .get(); - Settings indexSettings = getIndexResponse.settings().get("test-idx-1"); - verifyRemoteStoreIndexSettings( - indexSettings, - null, - null, - null, - ReplicationType.DOCUMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL + IllegalArgumentException exc = expectThrows( + IllegalArgumentException.class, + () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() + ); + assertThat( + exc.getMessage(), + containsString( + String.format( + Locale.ROOT, + "To enable %s, %s should be set to %s", + SETTING_REMOTE_STORE_ENABLED, + SETTING_REPLICATION_TYPE, + ReplicationType.SEGMENT + ) + ) ); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 728bac647d74a..76156e8234f3e 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -91,7 +91,6 @@ import org.opensearch.indices.InvalidIndexNameException; import org.opensearch.indices.ShardLimitValidator; import org.opensearch.indices.SystemIndices; -import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; @@ -105,7 +104,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -954,44 +952,19 @@ private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, if (CLUSTER_REMOTE_STORE_ENABLED_SETTING.get(clusterSettings)) { // Verify if we can create a remote store based index based on user provided settings if (canCreateRemoteStoreIndex(requestSettings) == false) { - return; + throw new IllegalArgumentException("Cannot override settings related to remote store."); } - // Verify index has replication type as SEGMENT - if (ReplicationType.DOCUMENT.equals(ReplicationType.parseString(settingsBuilder.get(SETTING_REPLICATION_TYPE)))) { - throw new IllegalArgumentException( - "Cannot enable [" - + SETTING_REMOTE_STORE_ENABLED - + "] when [" - + SETTING_REPLICATION_TYPE - + "] is " - + ReplicationType.DOCUMENT - ); - } - - settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true); - String remoteStoreRepo; - if (Objects.equals(requestSettings.get(INDEX_REMOTE_STORE_ENABLED_SETTING.getKey()), "true")) { - remoteStoreRepo = requestSettings.get(INDEX_REMOTE_STORE_REPOSITORY_SETTING.getKey()); - } else { - remoteStoreRepo = CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.get(clusterSettings); - } - settingsBuilder.put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo) - .put( - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, - requestSettings.get( - INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), - CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(clusterSettings) - ) - ); + settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) + .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.get(clusterSettings)) + .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(clusterSettings)); } } private static boolean canCreateRemoteStoreIndex(Settings requestSettings) { - return (INDEX_REPLICATION_TYPE_SETTING.exists(requestSettings) == false - || INDEX_REPLICATION_TYPE_SETTING.get(requestSettings).equals(ReplicationType.SEGMENT)) - && (INDEX_REMOTE_STORE_ENABLED_SETTING.exists(requestSettings) == false - || INDEX_REMOTE_STORE_ENABLED_SETTING.get(requestSettings)); + return INDEX_REMOTE_STORE_ENABLED_SETTING.exists(requestSettings) == false + && INDEX_REMOTE_STORE_REPOSITORY_SETTING.exists(requestSettings) == false + && INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.exists(requestSettings) == false; } public static void validateStoreTypeSettings(Settings settings) { diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index e52237c8dba99..025e21ffb08cc 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -1296,24 +1296,20 @@ public void testRemoteStoreDisabledByUserIndexSettings() { final Settings.Builder requestSettings = Settings.builder(); requestSettings.put(SETTING_REMOTE_STORE_ENABLED, false); request.settings(requestSettings.build()); - Settings indexSettings = aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - settings, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() - ); - verifyRemoteStoreIndexSettings( - indexSettings, - "false", - null, - null, - ReplicationType.SEGMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL + IllegalArgumentException exc = expectThrows( + IllegalArgumentException.class, + () -> aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + settings, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet() + ) ); + assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); } public void testRemoteStoreOverrideSegmentRepoIndexSettings() { @@ -1331,24 +1327,20 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { .put(SETTING_REMOTE_STORE_ENABLED, true) .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo"); request.settings(requestSettings.build()); - Settings indexSettings = aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - settings, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() - ); - verifyRemoteStoreIndexSettings( - indexSettings, - "true", - "my-custom-repo", - "my-translog-repo-1", - ReplicationType.SEGMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL + IllegalArgumentException exc = expectThrows( + IllegalArgumentException.class, + () -> aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + settings, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet() + ) ); + assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); } public void testRemoteStoreOverrideTranslogRepoIndexSettings() { @@ -1364,24 +1356,20 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() { final Settings.Builder requestSettings = Settings.builder(); requestSettings.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo"); request.settings(requestSettings.build()); - Settings indexSettings = aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - settings, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() - ); - verifyRemoteStoreIndexSettings( - indexSettings, - "true", - "my-segment-repo-1", - "my-custom-repo", - ReplicationType.SEGMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL + IllegalArgumentException exc = expectThrows( + IllegalArgumentException.class, + () -> aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + settings, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet() + ) ); + assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); } public void testRemoteStoreOverrideReplicationTypeIndexSettings() { @@ -1397,24 +1385,20 @@ public void testRemoteStoreOverrideReplicationTypeIndexSettings() { final Settings.Builder requestSettings = Settings.builder(); requestSettings.put(SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT); request.settings(requestSettings.build()); - Settings indexSettings = aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - settings, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() - ); - verifyRemoteStoreIndexSettings( - indexSettings, - null, - null, - null, - ReplicationType.DOCUMENT.toString(), - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL + IllegalArgumentException exc = expectThrows( + IllegalArgumentException.class, + () -> aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + settings, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet() + ) ); + assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); } public void testBuildIndexMetadata() { From d1d932910a6cc77f594b4ec1e91e66c95b11a115 Mon Sep 17 00:00:00 2001 From: bansvaru Date: Fri, 21 Jul 2023 12:49:27 +0530 Subject: [PATCH 2/9] fix failing tests Signed-off-by: bansvaru --- ...ateRemoteIndexClusterDefaultDocRepIT.java} | 2 +- .../metadata/MetadataCreateIndexService.java | 9 ++- .../MetadataCreateIndexServiceTests.java | 58 ------------------- 3 files changed, 8 insertions(+), 61 deletions(-) rename server/src/internalClusterTest/java/org/opensearch/remotestore/{CreateRemoteIndexClusterDefaultDocRep.java => CreateRemoteIndexClusterDefaultDocRepIT.java} (97%) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRepIT.java similarity index 97% rename from server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java rename to server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRepIT.java index fbe821d049a17..32c02332e05b2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRep.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRepIT.java @@ -25,7 +25,7 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) -public class CreateRemoteIndexClusterDefaultDocRep extends CreateRemoteIndexIT { +public class CreateRemoteIndexClusterDefaultDocRepIT extends CreateRemoteIndexIT { @Override protected Settings nodeSettings(int nodeOriginal) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 76156e8234f3e..3a10f7ad177c2 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -949,8 +949,8 @@ private static void updateReplicationStrategy(Settings.Builder settingsBuilder, * @param clusterSettings cluster level settings */ private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, Settings requestSettings, Settings clusterSettings) { - if (CLUSTER_REMOTE_STORE_ENABLED_SETTING.get(clusterSettings)) { - // Verify if we can create a remote store based index based on user provided settings + if (CLUSTER_REMOTE_STORE_ENABLED_SETTING.get(clusterSettings) == true) { + // User should not be able to override remote store index settings if (canCreateRemoteStoreIndex(requestSettings) == false) { throw new IllegalArgumentException("Cannot override settings related to remote store."); } @@ -958,6 +958,11 @@ private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.get(clusterSettings)) .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(clusterSettings)); + } else { + // User should not be able to create remote indices + if (canCreateRemoteStoreIndex(requestSettings) == false) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Cannot create remote store indices where [%s] is not set", CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey())); + } } } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 025e21ffb08cc..0e6c1bf8717f5 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -1191,35 +1191,6 @@ public void testvalidateIndexSettings() { threadPool.shutdown(); } - public void testRemoteStoreNoUserOverrideConflictingReplicationTypeIndexSettings() { - Settings settings = Settings.builder() - .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") - .build(); - FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - IllegalArgumentException exc = expectThrows( - IllegalArgumentException.class, - () -> aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - settings, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() - ) - ); - assertThat( - exc.getMessage(), - containsString("Cannot enable [index.remote_store.enabled] when [index.replication.type] is DOCUMENT") - ); - } - public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettings() { Settings settings = Settings.builder() .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT) @@ -1372,35 +1343,6 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() { assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); } - public void testRemoteStoreOverrideReplicationTypeIndexSettings() { - Settings settings = Settings.builder() - .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") - .build(); - FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - final Settings.Builder requestSettings = Settings.builder(); - requestSettings.put(SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT); - request.settings(requestSettings.build()); - IllegalArgumentException exc = expectThrows( - IllegalArgumentException.class, - () -> aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - settings, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() - ) - ); - assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); - } - public void testBuildIndexMetadata() { IndexMetadata sourceIndexMetadata = IndexMetadata.builder("parent") .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) From c36056a266ef5db518ddba61507adc992a837659 Mon Sep 17 00:00:00 2001 From: bansvaru Date: Fri, 21 Jul 2023 13:13:15 +0530 Subject: [PATCH 3/9] refactor exception flow to improve error logging Signed-off-by: bansvaru --- .../remotestore/CreateRemoteIndexIT.java | 53 +++++++++++++++++-- .../cluster/metadata/IndexMetadata.java | 8 +-- .../metadata/MetadataCreateIndexService.java | 37 +++++++------ .../common/settings/IndexScopedSettings.java | 2 +- .../MetadataCreateIndexServiceTests.java | 38 +++++++++++-- .../opensearch/index/IndexSettingsTests.java | 6 +-- 6 files changed, 114 insertions(+), 30 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java index 2d2533500bf9d..24882b03ef721 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java @@ -115,7 +115,17 @@ public void testRemoteStoreDisabledByUser() throws Exception { IllegalArgumentException.class, () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() ); - assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); + assertThat( + exc.getMessage(), + containsString( + String.format( + Locale.ROOT, + "Cannot override [%s] settings when [%s] is set to [true].", + SETTING_REMOTE_STORE_ENABLED, + CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() + ) + ) + ); } public void testRemoteStoreEnabledByUserWithoutRemoteRepoAndSegmentReplicationIllegalArgumentException() throws Exception { @@ -147,7 +157,17 @@ public void testRemoteStoreEnabledByUserWithoutRemoteRepoIllegalArgumentExceptio IllegalArgumentException.class, () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() ); - assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); + assertThat( + exc.getMessage(), + containsString( + String.format( + Locale.ROOT, + "Cannot override [%s] settings when [%s] is set to [true].", + SETTING_REMOTE_STORE_ENABLED, + CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() + ) + ) + ); } public void testReplicationTypeDocumentByUser() throws Exception { @@ -197,7 +217,7 @@ public void testRemoteStoreSegmentRepoWithoutRemoteEnabledAndSegmentReplicationI ); } - public void testRemoteStoreEnabledByUserWithRemoteRepo() throws Exception { + public void testRemoteStoreEnabledByUserWithRemoteRepoIllegalArgumentException() throws Exception { Settings settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) @@ -210,7 +230,18 @@ public void testRemoteStoreEnabledByUserWithRemoteRepo() throws Exception { IllegalArgumentException.class, () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() ); - assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); + assertThat( + exc.getMessage(), + containsString( + String.format( + Locale.ROOT, + "Cannot override [%s][%s] settings when [%s] is set to [true].", + SETTING_REMOTE_STORE_ENABLED, + SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() + ) + ) + ); } public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() throws Exception { @@ -249,7 +280,19 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception { IllegalArgumentException.class, () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() ); - assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); + assertThat( + exc.getMessage(), + containsString( + String.format( + Locale.ROOT, + "Cannot override [%s][%s][%s] settings when [%s] is set to [true].", + SETTING_REMOTE_STORE_ENABLED, + SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, + CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() + ) + ) + ); } public void testRemoteStoreOverrideReplicationTypeIndexSettings() throws Exception { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 1ba38daa40566..238a118e06758 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -334,7 +334,7 @@ public Iterator> settings() { /** * Used to specify remote store repository to use for this index. */ - public static final Setting INDEX_REMOTE_STORE_REPOSITORY_SETTING = Setting.simpleString( + public static final Setting INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING = Setting.simpleString( SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, new Setting.Validator<>() { @@ -345,10 +345,12 @@ public void validate(final String value) {} public void validate(final String value, final Map, Object> settings) { if (value == null || value.isEmpty()) { throw new IllegalArgumentException( - "Setting " + INDEX_REMOTE_STORE_REPOSITORY_SETTING.getKey() + " should be provided with non-empty repository ID" + "Setting " + + INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey() + + " should be provided with non-empty repository ID" ); } else { - validateRemoteStoreSettingEnabled(settings, INDEX_REMOTE_STORE_REPOSITORY_SETTING); + validateRemoteStoreSettingEnabled(settings, INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING); } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 3a10f7ad177c2..294e0d7e06bd3 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -114,12 +114,13 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; import static java.util.stream.Collectors.toList; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REMOTE_STORE_REPOSITORY_SETTING; +import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REPLICATION_TYPE_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; @@ -949,27 +950,33 @@ private static void updateReplicationStrategy(Settings.Builder settingsBuilder, * @param clusterSettings cluster level settings */ private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, Settings requestSettings, Settings clusterSettings) { - if (CLUSTER_REMOTE_STORE_ENABLED_SETTING.get(clusterSettings) == true) { - // User should not be able to override remote store index settings - if (canCreateRemoteStoreIndex(requestSettings) == false) { - throw new IllegalArgumentException("Cannot override settings related to remote store."); - } + boolean isRemoteStoreClusterEnabled = CLUSTER_REMOTE_STORE_ENABLED_SETTING.get(clusterSettings); + List overriddenSettings = getRemoteStoreOverriddenSetting(requestSettings); + if (overriddenSettings.isEmpty() == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Cannot override [%s] settings when [%s] is set to [%s].", + String.join("][", overriddenSettings), + CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), + isRemoteStoreClusterEnabled + ) + ); + } + if (isRemoteStoreClusterEnabled == true) { settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.get(clusterSettings)) .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(clusterSettings)); - } else { - // User should not be able to create remote indices - if (canCreateRemoteStoreIndex(requestSettings) == false) { - throw new IllegalArgumentException(String.format(Locale.ROOT, "Cannot create remote store indices where [%s] is not set", CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey())); - } } } - private static boolean canCreateRemoteStoreIndex(Settings requestSettings) { - return INDEX_REMOTE_STORE_ENABLED_SETTING.exists(requestSettings) == false - && INDEX_REMOTE_STORE_REPOSITORY_SETTING.exists(requestSettings) == false - && INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.exists(requestSettings) == false; + private static List getRemoteStoreOverriddenSetting(Settings requestSettings) { + return Stream.of( + INDEX_REMOTE_STORE_ENABLED_SETTING, + INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING, + INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING + ).filter(setting -> setting.exists(requestSettings)).map(Setting::getKey).collect(toList()); } public static void validateStoreTypeSettings(Settings settings) { diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 3cc7c351fe1bf..be2b5f00bc0ec 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -235,7 +235,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { FeatureFlags.REMOTE_STORE, List.of( IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING, - IndexMetadata.INDEX_REMOTE_STORE_REPOSITORY_SETTING, + IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING, IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING ), FeatureFlags.CONCURRENT_SEGMENT_SEARCH, diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 0e6c1bf8717f5..5384bf942f924 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -101,6 +101,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.Locale; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; @@ -1280,7 +1281,17 @@ public void testRemoteStoreDisabledByUserIndexSettings() { Collections.emptySet() ) ); - assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); + assertThat( + exc.getMessage(), + containsString( + String.format( + Locale.ROOT, + "Cannot override [%s] settings when [%s] is set to [true].", + SETTING_REMOTE_STORE_ENABLED, + CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() + ) + ) + ); } public void testRemoteStoreOverrideSegmentRepoIndexSettings() { @@ -1311,7 +1322,18 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { Collections.emptySet() ) ); - assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); + assertThat( + exc.getMessage(), + containsString( + String.format( + Locale.ROOT, + "Cannot override [%s][%s] settings when [%s] is set to [true].", + SETTING_REMOTE_STORE_ENABLED, + SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() + ) + ) + ); } public void testRemoteStoreOverrideTranslogRepoIndexSettings() { @@ -1340,7 +1362,17 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() { Collections.emptySet() ) ); - assertThat(exc.getMessage(), containsString("Cannot override settings related to remote store.")); + assertThat( + exc.getMessage(), + containsString( + String.format( + Locale.ROOT, + "Cannot override [%s] settings when [%s] is set to [true].", + SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, + CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() + ) + ) + ); } public void testBuildIndexMetadata() { diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index e42e9b4970081..6207d2f2725ad 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -856,7 +856,7 @@ public void testRemoteRepositoryExplicitSetting() { public void testUpdateRemoteRepositoryFails() { Set> remoteStoreSettingSet = new HashSet<>(); - remoteStoreSettingSet.add(IndexMetadata.INDEX_REMOTE_STORE_REPOSITORY_SETTING); + remoteStoreSettingSet.add(IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING); IndexScopedSettings settings = new IndexScopedSettings(Settings.EMPTY, remoteStoreSettingSet); SettingsException error = expectThrows( SettingsException.class, @@ -881,7 +881,7 @@ public void testSetRemoteRepositoryFailsWhenRemoteStoreIsNotEnabled() { .build(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, - () -> IndexMetadata.INDEX_REMOTE_STORE_REPOSITORY_SETTING.get(indexSettings) + () -> IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(indexSettings) ); assertEquals( String.format( @@ -902,7 +902,7 @@ public void testSetRemoteRepositoryFailsWhenEmptyString() { .build(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, - () -> IndexMetadata.INDEX_REMOTE_STORE_REPOSITORY_SETTING.get(indexSettings) + () -> IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(indexSettings) ); assertEquals( String.format( From 80ee26a5065ea8bd03cfcc4d46dc923515c1ca8a Mon Sep 17 00:00:00 2001 From: bansvaru Date: Sat, 22 Jul 2023 08:35:50 +0530 Subject: [PATCH 4/9] fix failing tests Signed-off-by: bansvaru --- .../index/SegmentReplicationPressureIT.java | 4 +- ...emoteStoreMockRepositoryIntegTestCase.java | 6 +- .../remotestore/CreateRemoteIndexIT.java | 9 +- .../remotestore/PrimaryTermValidationIT.java | 1 + .../remotestore/RemoteIndexRecoveryIT.java | 99 +++- .../remotestore/RemoteRestoreSnapshotIT.java | 530 ++++++++++++++++++ .../RemoteStoreBaseIntegTestCase.java | 44 +- .../ReplicaToPrimaryPromotionIT.java | 6 +- .../SegmentReplicationUsingRemoteStoreIT.java | 15 +- ...tReplicationWithRemoteStorePressureIT.java | 18 +- .../RemoteStoreMultipartFileCorruptionIT.java | 52 +- .../opensearch/snapshots/CloneSnapshotIT.java | 27 +- .../snapshots/DeleteSnapshotIT.java | 64 +-- .../RemoteIndexSnapshotStatusApiIT.java | 209 +++++++ .../snapshots/RestoreSnapshotIT.java | 505 ----------------- .../snapshots/SnapshotStatusApisIT.java | 99 +--- .../metadata/MetadataCreateIndexService.java | 8 +- .../common/settings/ClusterSettings.java | 2 +- .../opensearch/indices/IndicesService.java | 2 +- .../opensearch/snapshots/RestoreService.java | 7 +- .../MetadataCreateIndexServiceTests.java | 12 +- .../BlobStoreRepositoryHelperTests.java | 140 +++++ .../BlobStoreRepositoryRemoteIndexTests.java | 373 ++++++++++++ .../blobstore/BlobStoreRepositoryTests.java | 417 +------------- .../AbstractSnapshotIntegTestCase.java | 5 +- 25 files changed, 1477 insertions(+), 1177 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java create mode 100644 server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java create mode 100644 server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java b/server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java index 60ff82e617dbd..cf73c370cce8f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java @@ -15,6 +15,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.lease.Releasable; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; @@ -30,6 +31,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -260,7 +262,7 @@ public void testFailStaleReplica() throws Exception { public void testWithDocumentReplicationEnabledIndex() throws Exception { assumeTrue( "Can't create DocRep index with remote store enabled. Skipping.", - indexSettings().getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) == false + Objects.equals(featureFlagSettings().get(FeatureFlags.REMOTE_STORE, "false"), "false") ); Settings settings = Settings.builder().put(MAX_REPLICATION_TIME_SETTING.getKey(), TimeValue.timeValueMillis(500)).build(); // Starts a primary and replica node. diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index 709c027c3f347..5bfbbc11da77d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; public abstract class AbstractRemoteStoreMockRepositoryIntegTestCase extends AbstractSnapshotIntegTestCase { @@ -46,7 +47,7 @@ protected Settings featureFlagSettings() { public void setup() { FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); FeatureFlagSetter.set(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL); - internalCluster().startClusterManagerOnlyNode(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REPOSITORY_NAME, TRANSLOG_REPOSITORY_NAME)); } @Override @@ -62,9 +63,6 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas) { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, TRANSLOG_REPOSITORY_NAME) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java index 24882b03ef721..fc583942c6df4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java @@ -29,10 +29,8 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_REPOSITORY_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) @@ -49,10 +47,7 @@ public void teardown() { protected Settings nodeSettings(int nodeOriginal) { Settings settings = super.nodeSettings(nodeOriginal); Settings.Builder builder = Settings.builder() - .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") + .put(remoteStoreClusterSettings("my-segment-repo-1", "my-translog-repo-1", true)) .put(settings); return builder.build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java index 9d63c9b528314..ee32c880257d1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java @@ -61,6 +61,7 @@ public void testPrimaryTermValidation() throws Exception { .put(FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), "1s") .put(FollowersChecker.FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), "1s") .put(FollowersChecker.FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), 1) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, REPOSITORY_2_NAME, true)) .build(); internalCluster().startClusterManagerOnlyNode(clusterSettings); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java index 4f7961cec22d7..d92ac83544a25 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java @@ -23,15 +23,21 @@ import java.nio.file.Path; +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteIndexRecoveryIT extends IndexRecoveryIT { - protected static final String REPOSITORY_NAME = "test-remore-store-repo"; + protected static final String REPOSITORY_NAME = "test-remote-store-repo"; protected Path absolutePath; + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(remoteStoreClusterSettings(REPOSITORY_NAME)).build(); + } + @Override protected Settings featureFlagSettings() { return Settings.builder() @@ -57,9 +63,6 @@ public Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) - .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); @@ -81,7 +84,91 @@ protected int numDocs() { } @Override - protected boolean shouldAssertOngoingRecoveryInRerouteRecovery() { - return false; + public void testUsesFileBasedRecoveryIfRetentionLeaseMissing() { + // Retention lease based tests not applicable for remote store; + } + + @Override + public void testPeerRecoveryTrimsLocalTranslog() { + // Peer recovery usecase not valid for remote enabled indices + } + + @Override + public void testHistoryRetention() { + // History retention not applicable for remote store + } + + @Override + public void testUsesFileBasedRecoveryIfOperationsBasedRecoveryWouldBeUnreasonable() { + // History retention not applicable for remote store + } + + @Override + public void testUsesFileBasedRecoveryIfRetentionLeaseAheadOfGlobalCheckpoint() { + // History retention not applicable for remote store + } + + @Override + public void testRecoverLocallyUpToGlobalCheckpoint() { + // History retention not applicable for remote store + } + + @Override + public void testCancelNewShardRecoveryAndUsesExistingShardCopy() { + // History retention not applicable for remote store + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8919") + @Override + public void testReservesBytesDuringPeerRecoveryPhaseOne() { + + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8919") + @Override + public void testAllocateEmptyPrimaryResetsGlobalCheckpoint() { + + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8919") + @Override + public void testDoesNotCopyOperationsInSafeCommit() { + + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8919") + @Override + public void testRepeatedRecovery() { + + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8919") + @Override + public void testDisconnectsWhileRecovering() { + + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8919") + @Override + public void testTransientErrorsDuringRecoveryAreRetried() { + + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8919") + @Override + public void testDoNotInfinitelyWaitForMapping() { + + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8919") + @Override + public void testDisconnectsDuringRecovery() { + + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8919") + @Override + public void testReplicaRecovery() { + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java new file mode 100644 index 0000000000000..8c33bf36ad45d --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -0,0 +1,530 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore; + +import org.junit.After; +import org.junit.Before; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; +import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.opensearch.action.admin.indices.get.GetIndexRequest; +import org.opensearch.action.admin.indices.get.GetIndexResponse; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.client.Client; +import org.opensearch.client.Requests; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.io.PathUtils; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexSettings; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.snapshots.AbstractSnapshotIntegTestCase; +import org.opensearch.snapshots.SnapshotState; +import org.opensearch.test.InternalTestCluster; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.concurrent.ExecutionException; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +public class RemoteRestoreSnapshotIT extends AbstractSnapshotIntegTestCase { + private static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; + private Path remoteRepoPath; + + @Before + public void setup() { + remoteRepoPath = randomRepoPath().toAbsolutePath(); + createRepository(BASE_REMOTE_REPO, "fs", remoteRepoPath); + } + + @After + public void teardown() { + assertAcked(clusterAdmin().prepareDeleteRepository(BASE_REMOTE_REPO)); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.REMOTE_STORE, "true") + .put(remoteStoreClusterSettings(BASE_REMOTE_REPO)) + .build(); + } + + private Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) { + Settings.Builder settingsBuilder = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s"); + return settingsBuilder; + } + + private void indexDocuments(Client client, String indexName, int numOfDocs) { + indexDocuments(client, indexName, 0, numOfDocs); + } + + private void indexDocuments(Client client, String indexName, int fromId, int toId) { + for (int i = fromId; i < toId; i++) { + String id = Integer.toString(i); + client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); + } + client.admin().indices().prepareFlush(indexName).get(); + } + + private void assertDocsPresentInIndex(Client client, String indexName, int numOfDocs) { + for (int i = 0; i < numOfDocs; i++) { + String id = Integer.toString(i); + logger.info("checking for index " + indexName + " with docId" + id); + assertTrue("doc with id" + id + " is not present for index " + indexName, client.prepareGet(indexName, id).get().isExists()); + } + } + + public void testRestoreOperationsShallowCopyEnabled() throws IOException, ExecutionException, InterruptedException { + String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); + String primary = internalCluster().startDataOnlyNode(); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String snapshotName1 = "test-restore-snapshot1"; + String snapshotName2 = "test-restore-snapshot2"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + String restoredIndexName1 = indexName1 + "-restored"; + String restoredIndexName1Seg = indexName1 + "-restored-seg"; + String restoredIndexName1Doc = indexName1 + "-restored-doc"; + String restoredIndexName2 = indexName2 + "-restored"; + + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true)); + + Client client = client(); + Settings indexSettings = getIndexSettings(1, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(1, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 5; + final int numDocsInIndex2 = 6; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + internalCluster().startDataOnlyNode(); + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1, indexName2) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat( + createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + updateRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); + CreateSnapshotResponse createSnapshotResponse2 = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(true) + .setIndices(indexName1, indexName2) + .get(); + assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat( + createSnapshotResponse2.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse2.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + DeleteResponse deleteResponse = client().prepareDelete(indexName1, "0").execute().actionGet(); + assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); + indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + randomIntBetween(2, 5)); + ensureGreen(indexName1); + + RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1) + .get(); + RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(false) + .setIndices(indexName2) + .setRenamePattern(indexName2) + .setRenameReplacement(restoredIndexName2) + .get(); + assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); + assertEquals(restoreSnapshotResponse2.status(), RestStatus.ACCEPTED); + ensureGreen(restoredIndexName1, restoredIndexName2); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); + assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2); + + // deleting data for restoredIndexName1 and restoring from remote store. + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); + ensureRed(restoredIndexName1); + // Re-initialize client to make sure we are not using client from stopped node. + client = client(clusterManagerNode); + assertAcked(client.admin().indices().prepareClose(restoredIndexName1)); + client.admin() + .cluster() + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true), + PlainActionFuture.newFuture() + ); + ensureYellowAndNoInitializingShards(restoredIndexName1); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1); + // indexing some new docs and validating + indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); + + // restore index as seg rep enabled with remote store and remote translog disabled + RestoreSnapshotResponse restoreSnapshotResponse3 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1Seg) + .get(); + assertEquals(restoreSnapshotResponse3.status(), RestStatus.ACCEPTED); + ensureGreen(restoredIndexName1Seg); + + GetIndexResponse getIndexResponse = client.admin() + .indices() + .getIndex(new GetIndexRequest().indices(restoredIndexName1Seg).includeDefaults(true)) + .get(); + indexSettings = getIndexResponse.settings().get(restoredIndexName1Seg); + assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); + assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null)); + assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE)); + assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1); + // indexing some new docs and validating + indexDocuments(client, restoredIndexName1Seg, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1Seg); + assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1 + 2); + + // restore index as doc rep based from shallow copy snapshot + RestoreSnapshotResponse restoreSnapshotResponse4 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, IndexMetadata.SETTING_REPLICATION_TYPE) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1Doc) + .get(); + assertEquals(restoreSnapshotResponse4.status(), RestStatus.ACCEPTED); + ensureGreen(restoredIndexName1Doc); + + getIndexResponse = client.admin() + .indices() + .getIndex(new GetIndexRequest().indices(restoredIndexName1Doc).includeDefaults(true)) + .get(); + indexSettings = getIndexResponse.settings().get(restoredIndexName1Doc); + assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); + assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null)); + assertNull(indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE)); + assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1); + // indexing some new docs and validating + indexDocuments(client, restoredIndexName1Doc, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1Doc); + assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1 + 2); + } + + public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { + String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); + String primary = internalCluster().startDataOnlyNode(); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String snapshotName1 = "test-restore-snapshot1"; + String snapshotName2 = "test-restore-snapshot2"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + String restoredIndexName2 = indexName2 + "-restored"; + + boolean enableShallowCopy = randomBoolean(); + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, enableShallowCopy)); + + Client client = client(); + Settings indexSettings = getIndexSettings(1, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(1, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 5; + final int numDocsInIndex2 = 6; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + internalCluster().startDataOnlyNode(); + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1, indexName2) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat( + createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + updateRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); + CreateSnapshotResponse createSnapshotResponse2 = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(true) + .setIndices(indexName1, indexName2) + .get(); + assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat( + createSnapshotResponse2.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse2.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + DeleteResponse deleteResponse = client().prepareDelete(indexName1, "0").execute().actionGet(); + assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); + indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + randomIntBetween(2, 5)); + ensureGreen(indexName1); + + assertAcked(client().admin().indices().prepareClose(indexName1)); + + RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIndices(indexName1) + .get(); + RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(false) + .setIndices(indexName2) + .setRenamePattern(indexName2) + .setRenameReplacement(restoredIndexName2) + .get(); + assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); + assertEquals(restoreSnapshotResponse2.status(), RestStatus.ACCEPTED); + ensureGreen(indexName1, restoredIndexName2); + assertDocsPresentInIndex(client, indexName1, numDocsInIndex1); + assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2); + + // deleting data for restoredIndexName1 and restoring from remote store. + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); + ensureRed(indexName1); + // Re-initialize client to make sure we are not using client from stopped node. + client = client(clusterManagerNode); + assertAcked(client.admin().indices().prepareClose(indexName1)); + client.admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1).restoreAllShards(true), PlainActionFuture.newFuture()); + ensureYellowAndNoInitializingShards(indexName1); + ensureGreen(indexName1); + assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); + // indexing some new docs and validating + indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(indexName1); + assertDocsPresentInIndex(client, indexName1, numDocsInIndex1 + 2); + } + + public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException { + String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); + String primary = internalCluster().startDataOnlyNode(); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String remoteStoreRepo2Name = "test-rs-repo-2" + TEST_REMOTE_STORE_REPO_SUFFIX; + String snapshotName1 = "test-restore-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + Path absolutePath3 = randomRepoPath().toAbsolutePath(); + String restoredIndexName1 = indexName1 + "-restored"; + + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); + createRepository(remoteStoreRepo2Name, "fs", absolutePath3); + + Client client = client(); + Settings indexSettings = getIndexSettings(1, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(1, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 5; + final int numDocsInIndex2 = 6; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + internalCluster().startDataOnlyNode(); + + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1, indexName2) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat( + createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + Settings remoteStoreIndexSettings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo2Name) + .build(); + // restore index as a remote store index with different remote store repo + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(remoteStoreIndexSettings) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1) + .get(); + assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1); + + // deleting data for restoredIndexName1 and restoring from remote store. + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); + // Re-initialize client to make sure we are not using client from stopped node. + client = client(clusterManagerNode); + assertAcked(client.admin().indices().prepareClose(restoredIndexName1)); + client.admin() + .cluster() + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true), + PlainActionFuture.newFuture() + ); + ensureYellowAndNoInitializingShards(restoredIndexName1); + ensureGreen(restoredIndexName1); + // indexing some new docs and validating + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); + indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); + } + + public void testRestoreShallowSnapshotRepositoryOverriden() throws ExecutionException, InterruptedException { + String indexName1 = "testindex1"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String remoteStoreRepoNameUpdated = "test-rs-repo-updated" + TEST_REMOTE_STORE_REPO_SUFFIX; + String snapshotName1 = "test-restore-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + Path absolutePath2 = randomRepoPath().toAbsolutePath(); + String[] pathTokens = absolutePath1.toString().split("/"); + String basePath = pathTokens[pathTokens.length - 1]; + Arrays.copyOf(pathTokens, pathTokens.length - 1); + Path location = PathUtils.get(String.join("/", pathTokens)); + pathTokens = absolutePath2.toString().split("/"); + String basePath2 = pathTokens[pathTokens.length - 1]; + Arrays.copyOf(pathTokens, pathTokens.length - 1); + Path location2 = PathUtils.get(String.join("/", pathTokens)); + logger.info("Path 1 [{}]", absolutePath1); + logger.info("Path 2 [{}]", absolutePath2); + String restoredIndexName1 = indexName1 + "-restored"; + + createRepository(snapshotRepoName, "fs", getRepositorySettings(location, basePath, true)); + + Client client = client(); + Settings indexSettings = Settings.builder() + .put(super.indexSettings()) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + createIndex(indexName1, indexSettings); + + int numDocsInIndex1 = randomIntBetween(2, 5); + indexDocuments(client, indexName1, numDocsInIndex1); + + ensureGreen(indexName1); + + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat( + createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) + ); + assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); + + createRepository(BASE_REMOTE_REPO, "fs", absolutePath2); + + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1) + .get(); + + assertTrue(restoreSnapshotResponse.getRestoreInfo().failedShards() > 0); + + ensureRed(restoredIndexName1); + + client().admin().indices().close(Requests.closeIndexRequest(restoredIndexName1)).get(); + createRepository(remoteStoreRepoNameUpdated, "fs", remoteRepoPath); + RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1) + .setSourceRemoteStoreRepository(remoteStoreRepoNameUpdated) + .get(); + + assertTrue(restoreSnapshotResponse2.getRestoreInfo().failedShards() == 0); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); + + // indexing some new docs and validating + indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); + } + +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 2887fbc56106c..50c3fe7261edf 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -29,11 +29,14 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; +import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; +import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { - protected static final String REPOSITORY_NAME = "test-remore-store-repo"; - protected static final String REPOSITORY_2_NAME = "test-remore-store-repo-2"; + protected static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected static final String REPOSITORY_2_NAME = "test-remote-store-repo-2"; protected static final int SHARD_COUNT = 1; protected static final int REPLICA_COUNT = 1; protected Path absolutePath; @@ -51,6 +54,14 @@ protected boolean addMockInternalEngine() { return false; } + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, REPOSITORY_2_NAME, true)) + .build(); + } + @Override protected Settings featureFlagSettings() { return Settings.builder() @@ -64,21 +75,40 @@ public Settings indexSettings() { return defaultIndexSettings(); } - IndexResponse indexSingleDoc(String indexName) { + protected IndexResponse indexSingleDoc(String indexName) { return client().prepareIndex(indexName) .setId(UUIDs.randomBase64UUID()) .setSource(documentKeys.get(randomIntBetween(0, documentKeys.size() - 1)), randomAlphaOfLength(5)) .get(); } + public static Settings remoteStoreClusterSettings(String segmentRepoName) { + return remoteStoreClusterSettings(segmentRepoName, segmentRepoName); + } + + public static Settings remoteStoreClusterSettings( + String segmentRepoName, + String translogRepoName, + boolean randomizeSameRepoForRSSAndRTS + ) { + return remoteStoreClusterSettings( + segmentRepoName, + randomizeSameRepoForRSSAndRTS ? (randomBoolean() ? translogRepoName : segmentRepoName) : translogRepoName + ); + } + + public static Settings remoteStoreClusterSettings(String segmentRepoName, String translogRepoName) { + return Settings.builder() + .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) + .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), segmentRepoName) + .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), translogRepoName) + .build(); + } + private Settings defaultIndexSettings() { - boolean sameRepoForRSSAndRTS = randomBoolean(); return Settings.builder() .put(super.indexSettings()) .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) - .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, sameRepoForRSSAndRTS ? REPOSITORY_NAME : REPOSITORY_2_NAME) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, SHARD_COUNT) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, REPLICA_COUNT) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java index 6764c50175e61..549b4985894a7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java @@ -39,11 +39,7 @@ public void setup() { @Override public Settings indexSettings() { - return Settings.builder() - .put(super.indexSettings()) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shard_count) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) - .build(); + return Settings.builder().put(super.indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shard_count).build(); } public void testPromoteReplicaToPrimary() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java index f298fac7c894e..6f76c21cc0411 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java @@ -10,7 +10,6 @@ import org.junit.After; import org.junit.Before; -import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.indices.replication.SegmentReplicationIT; @@ -18,6 +17,7 @@ import java.nio.file.Path; +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; /** @@ -32,13 +32,12 @@ public class SegmentReplicationUsingRemoteStoreIT extends SegmentReplicationIT { private static final String REPOSITORY_NAME = "test-remote-store-repo"; @Override - public Settings indexSettings() { - return Settings.builder() - .put(super.indexSettings()) - .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) - .build(); + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(remoteStoreClusterSettings(REPOSITORY_NAME)).build(); + } + + protected boolean segmentReplicationWithRemoteEnabled() { + return true; } @Override diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java index 0b64680033d84..38db7a7c7269e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java @@ -10,15 +10,14 @@ import org.junit.After; import org.junit.Before; -import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.SegmentReplicationPressureIT; -import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchIntegTestCase; import java.nio.file.Path; +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; /** @@ -32,14 +31,8 @@ public class SegmentReplicationWithRemoteStorePressureIT extends SegmentReplicat private static final String REPOSITORY_NAME = "test-remote-store-repo"; @Override - public Settings indexSettings() { - return Settings.builder() - .put(super.indexSettings()) - .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .build(); + protected boolean segmentReplicationWithRemoteEnabled() { + return true; } @Override @@ -51,6 +44,11 @@ protected Settings featureFlagSettings() { .build(); } + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(remoteStoreClusterSettings(REPOSITORY_NAME)).build(); + } + @Before public void setup() { internalCluster().startClusterManagerOnlyNode(); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java index 529e84d281476..b801c28983890 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java @@ -8,31 +8,22 @@ package org.opensearch.remotestore.multipart; -import org.junit.After; import org.junit.Before; -import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.IndicesOptions; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexModule; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.plugins.Plugin; -import org.opensearch.remotestore.multipart.mocks.MockFsRepository; +import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; -import org.opensearch.test.OpenSearchIntegTestCase; -import java.nio.file.Path; import java.util.Collection; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +public class RemoteStoreMultipartFileCorruptionIT extends RemoteStoreBaseIntegTestCase { -public class RemoteStoreMultipartFileCorruptionIT extends OpenSearchIntegTestCase { - - protected static final String REPOSITORY_NAME = "test-remore-store-repo"; private static final String INDEX_NAME = "remote-store-test-idx-1"; @Override @@ -40,34 +31,9 @@ protected Collection> nodePlugins() { return Stream.concat(super.nodePlugins().stream(), Stream.of(MockFsRepositoryPlugin.class)).collect(Collectors.toList()); } - @Override - protected Settings featureFlagSettings() { - return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_STORE, "true").build(); - } - @Before public void setup() { - internalCluster().startClusterManagerOnlyNode(); - Path absolutePath = randomRepoPath().toAbsolutePath(); - putRepository(absolutePath); - } - - protected void putRepository(Path path) { - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME) - .setType(MockFsRepositoryPlugin.TYPE) - .setSettings( - Settings.builder() - .put("location", path) - // custom setting for MockFsRepositoryPlugin - .put(MockFsRepository.TRIGGER_DATA_INTEGRITY_FAILURE.getKey(), true) - ) - ); - } - - @After - public void teardown() { - assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); + setupRepo(); } protected Settings remoteStoreIndexSettings() { @@ -78,26 +44,16 @@ protected Settings remoteStoreIndexSettings() { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) .build(); } - private IndexResponse indexSingleDoc() { - return client().prepareIndex(INDEX_NAME) - .setId(UUIDs.randomBase64UUID()) - .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) - .get(); - } - public void testLocalFileCorruptionDuringUpload() { internalCluster().startDataOnlyNodes(1); createIndex(INDEX_NAME, remoteStoreIndexSettings()); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); - indexSingleDoc(); + indexSingleDoc(INDEX_NAME); client().admin() .indices() diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java index 3de982f89ac80..d19cb513ed38d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java @@ -54,6 +54,7 @@ import java.util.Collection; import java.util.List; +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; @@ -162,7 +163,8 @@ public void testCloneSnapshotIndex() throws Exception { public void testCloneShallowSnapshotIndex() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(); + final String remoteStoreRepoName = "remote-store-repo-name"; + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName)); internalCluster().startDataOnlyNode(); final String snapshotRepoName = "snapshot-repo-name"; @@ -174,14 +176,13 @@ public void testCloneShallowSnapshotIndex() throws Exception { createRepository(shallowSnapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath)); final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); final String remoteStoreEnabledIndexName = "remote-index-1"; - final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); @@ -209,7 +210,10 @@ public void testCloneShallowSnapshotIndex() throws Exception { public void testShallowCloneNameAvailability() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(LARGE_SNAPSHOT_POOL_SETTINGS); + final String remoteStoreRepoName = "remote-store-repo-name"; + internalCluster().startClusterManagerOnlyNode( + Settings.builder().put(LARGE_SNAPSHOT_POOL_SETTINGS).put(remoteStoreClusterSettings(remoteStoreRepoName)).build() + ); internalCluster().startDataOnlyNode(); final String shallowSnapshotRepoName = "shallow-snapshot-repo-name"; @@ -217,14 +221,13 @@ public void testShallowCloneNameAvailability() throws Exception { createRepository(shallowSnapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath)); final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); final String remoteStoreEnabledIndexName = "remote-index-1"; - final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); @@ -244,7 +247,8 @@ public void testShallowCloneNameAvailability() throws Exception { public void testCloneAfterRepoShallowSettingEnabled() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(); + final String remoteStoreRepoName = "remote-store-repo-name"; + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName)); internalCluster().startDataOnlyNode(); final String snapshotRepoName = "snapshot-repo-name"; @@ -252,14 +256,13 @@ public void testCloneAfterRepoShallowSettingEnabled() throws Exception { createRepository(snapshotRepoName, "fs", snapshotRepoPath); final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); final String remoteStoreEnabledIndexName = "remote-index-1"; - final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); @@ -281,7 +284,8 @@ public void testCloneAfterRepoShallowSettingEnabled() throws Exception { public void testCloneAfterRepoShallowSettingDisabled() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(); + final String remoteStoreRepoName = "remote-store-repo-name"; + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName)); internalCluster().startDataOnlyNode(); final String snapshotRepoName = "snapshot-repo-name"; @@ -289,14 +293,13 @@ public void testCloneAfterRepoShallowSettingDisabled() throws Exception { createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); final String remoteStoreEnabledIndexName = "remote-index-1"; - final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java index 2688449294f3d..b12fbdd2a9bd7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -25,15 +25,18 @@ import java.util.stream.Stream; import static org.hamcrest.Matchers.is; +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class DeleteSnapshotIT extends AbstractSnapshotIntegTestCase { + private static final String REMOTE_REPO_NAME = "remote-store-repo-name"; + public void testDeleteSnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); internalCluster().startDataOnlyNode(); final String snapshotRepoName = "snapshot-repo-name"; @@ -41,20 +44,19 @@ public void testDeleteSnapshot() throws Exception { createRepository(snapshotRepoName, "fs", snapshotRepoPath); final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); final String remoteStoreEnabledIndexName = "remote-index-1"; - final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); final String snapshot = "snapshot"; createFullSnapshot(snapshotRepoName, snapshot); - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == 0); assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 1); assertAcked(startDeleteSnapshot(snapshotRepoName, snapshot).get()); @@ -64,32 +66,31 @@ public void testDeleteSnapshot() throws Exception { public void testDeleteShallowCopySnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); internalCluster().startDataOnlyNode(); final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); final String remoteStoreEnabledIndexName = "remote-index-1"; - final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); final String shallowSnapshot = "shallow-snapshot"; createFullSnapshot(snapshotRepoName, shallowSnapshot); - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == 1); assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 1); assertAcked(startDeleteSnapshot(snapshotRepoName, shallowSnapshot).get()); assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 0); - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == 0); } // Deleting multiple shallow copy snapshots as part of single delete call with repo having only shallow copy snapshots. @@ -97,23 +98,22 @@ public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); internalCluster().startDataOnlyNode(); final Client clusterManagerClient = internalCluster().clusterManagerClient(); ensureStableCluster(2); + final Path remoteStoreRepoPath = randomRepoPath(); + createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); + final String snapshotRepoName = "snapshot-repo-name"; final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); final String testIndex = "index-test"; createIndexWithContent(testIndex); - final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String remoteStoreEnabledIndexName = "remote-index-1"; - final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); @@ -122,7 +122,7 @@ public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception { List shallowCopySnapshots = createNSnapshots(snapshotRepoName, totalShallowCopySnapshotsCount); List snapshotsToBeDeleted = shallowCopySnapshots.subList(0, randomIntBetween(2, totalShallowCopySnapshotsCount)); int tobeDeletedSnapshotsCount = snapshotsToBeDeleted.size(); - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == totalShallowCopySnapshotsCount); assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalShallowCopySnapshotsCount); // Deleting subset of shallow copy snapshots assertAcked( @@ -132,7 +132,7 @@ public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception { .get() ); assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalShallowCopySnapshotsCount - tobeDeletedSnapshotsCount); - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == totalShallowCopySnapshotsCount - tobeDeletedSnapshotsCount); } @@ -144,7 +144,7 @@ public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); final String dataNode = internalCluster().startDataOnlyNode(); ensureStableCluster(2); final String clusterManagerNode = internalCluster().getClusterManagerName(); @@ -156,11 +156,10 @@ public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception { createIndexWithContent(testIndex); final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); final String remoteStoreEnabledIndexName = "remote-index-1"; - final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); @@ -201,7 +200,7 @@ public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception { int totalSnapshotsCount = totalFullCopySnapshotsCount + totalShallowCopySnapshotsCount; - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == totalShallowCopySnapshotsCount); assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount); // Deleting subset of shallow copy snapshots assertAcked( @@ -213,7 +212,7 @@ public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception { totalSnapshotsCount -= tobeDeletedShallowCopySnapshotsCount; totalShallowCopySnapshotsCount -= tobeDeletedShallowCopySnapshotsCount; assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount); - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == totalShallowCopySnapshotsCount); // Deleting subset of full copy snapshots assertAcked( @@ -224,7 +223,7 @@ public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception { ); totalSnapshotsCount -= tobeDeletedFullCopySnapshotsCount; assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount); - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == totalShallowCopySnapshotsCount); } // Deleting subset of shallow and full copy snapshots as part of single delete call and then deleting all snapshots in the repo. @@ -233,7 +232,7 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); internalCluster().startDataOnlyNode(); final Client clusterManagerClient = internalCluster().clusterManagerClient(); ensureStableCluster(2); @@ -245,11 +244,10 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception { createIndexWithContent(testIndex); final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); final String remoteStoreEnabledIndexName = "remote-index-1"; - final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); @@ -268,7 +266,7 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception { int totalSnapshotsCount = totalFullCopySnapshotsCount + totalShallowCopySnapshotsCount; - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == totalShallowCopySnapshotsCount); assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount); // Deleting subset of shallow copy snapshots and full copy snapshots assertAcked( @@ -283,12 +281,12 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception { totalSnapshotsCount -= (tobeDeletedShallowCopySnapshotsCount + tobeDeletedFullCopySnapshotsCount); totalShallowCopySnapshotsCount -= tobeDeletedShallowCopySnapshotsCount; assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount); - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == totalShallowCopySnapshotsCount); // Deleting all the remaining snapshots assertAcked(clusterManagerClient.admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "*").get()); assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 0); - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == 0); } private List createNSnapshots(String repoName, int count) { diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java new file mode 100644 index 0000000000000..b6a5188c99335 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java @@ -0,0 +1,209 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.snapshots; + +import org.opensearch.action.ActionFuture; +import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStage; +import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStatus; +import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus; +import org.opensearch.cluster.SnapshotsInProgress; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.threadpool.ThreadPool; + +import java.nio.file.Path; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; + +public class RemoteIndexSnapshotStatusApiIT extends AbstractSnapshotIntegTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING.getKey(), 0) // We have tests that check by-timestamp order + .put(FeatureFlags.REMOTE_STORE, "true") + .put(remoteStoreClusterSettings("remote-store-repo-name")) + .build(); + } + + public void testStatusAPICallForShallowCopySnapshot() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used for the test"); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + + final String snapshotRepoName = "snapshot-repo-name"; + createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + ensureGreen(); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index(remoteStoreEnabledIndexName, "_doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + final String snapshot = "snapshot"; + createFullSnapshot(snapshotRepoName, snapshot); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); + + final SnapshotStatus snapshotStatus = getSnapshotStatus(snapshotRepoName, snapshot); + assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); + + // Validating that the incremental file count and incremental file size is zero for shallow copy + final SnapshotIndexShardStatus shallowSnapshotShardState = stateFirstShard(snapshotStatus, remoteStoreEnabledIndexName); + assertThat(shallowSnapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE)); + assertThat(shallowSnapshotShardState.getStats().getTotalFileCount(), greaterThan(0)); + assertThat(shallowSnapshotShardState.getStats().getTotalSize(), greaterThan(0L)); + assertThat(shallowSnapshotShardState.getStats().getIncrementalFileCount(), is(0)); + assertThat(shallowSnapshotShardState.getStats().getIncrementalSize(), is(0L)); + } + + public void testStatusAPIStatsForBackToBackShallowSnapshot() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used for the test"); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + + final String snapshotRepoName = "snapshot-repo-name"; + createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + ensureGreen(); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index(remoteStoreEnabledIndexName, "_doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + createFullSnapshot(snapshotRepoName, "test-snap-1"); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); + + SnapshotStatus snapshotStatus = getSnapshotStatus(snapshotRepoName, "test-snap-1"); + assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); + + SnapshotIndexShardStatus shallowSnapshotShardState = stateFirstShard(snapshotStatus, remoteStoreEnabledIndexName); + assertThat(shallowSnapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE)); + final int totalFileCount = shallowSnapshotShardState.getStats().getTotalFileCount(); + final long totalSize = shallowSnapshotShardState.getStats().getTotalSize(); + final int incrementalFileCount = shallowSnapshotShardState.getStats().getIncrementalFileCount(); + final long incrementalSize = shallowSnapshotShardState.getStats().getIncrementalSize(); + + createFullSnapshot(snapshotRepoName, "test-snap-2"); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 2); + + snapshotStatus = getSnapshotStatus(snapshotRepoName, "test-snap-2"); + assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); + shallowSnapshotShardState = stateFirstShard(snapshotStatus, remoteStoreEnabledIndexName); + assertThat(shallowSnapshotShardState.getStats().getTotalFileCount(), equalTo(totalFileCount)); + assertThat(shallowSnapshotShardState.getStats().getTotalSize(), equalTo(totalSize)); + assertThat(shallowSnapshotShardState.getStats().getIncrementalFileCount(), equalTo(incrementalFileCount)); + assertThat(shallowSnapshotShardState.getStats().getIncrementalSize(), equalTo(incrementalSize)); + } + + public void testStatusAPICallInProgressShallowSnapshot() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used for the test"); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + + final String snapshotRepoName = "snapshot-repo-name"; + createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy().put("block_on_data", true)); + + final Path remoteStoreRepoPath = randomRepoPath(); + final String remoteStoreRepoName = "remote-store-repo-name"; + createRepository(remoteStoreRepoName, "mock", remoteStoreRepoPath); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + ensureGreen(); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index(remoteStoreEnabledIndexName, "_doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + logger.info("--> snapshot"); + ActionFuture createSnapshotResponseActionFuture = startFullSnapshot(snapshotRepoName, "test-snap"); + + logger.info("--> wait for data nodes to get blocked"); + awaitNumberOfSnapshotsInProgress(1); + assertEquals( + SnapshotsInProgress.State.STARTED, + client().admin() + .cluster() + .prepareSnapshotStatus(snapshotRepoName) + .setSnapshots("test-snap") + .get() + .getSnapshots() + .get(0) + .getState() + ); + + logger.info("--> unblock all data nodes"); + unblockAllDataNodes(snapshotRepoName); + + logger.info("--> wait for snapshot to finish"); + createSnapshotResponseActionFuture.actionGet(); + } + + private static SnapshotIndexShardStatus stateFirstShard(SnapshotStatus snapshotStatus, String indexName) { + return snapshotStatus.getIndices().get(indexName).getShards().get(0); + } + + private static SnapshotStatus getSnapshotStatus(String repoName, String snapshotName) { + try { + return client().admin().cluster().prepareSnapshotStatus(repoName).setSnapshots(snapshotName).get().getSnapshots().get(0); + } catch (SnapshotMissingException e) { + throw new AssertionError(e); + } + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java index 30a836b41e29e..debb541fb8bc7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java @@ -33,45 +33,30 @@ package org.opensearch.snapshots; import org.opensearch.action.ActionFuture; -import org.opensearch.action.DocWriteResponse; -import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; -import org.opensearch.action.admin.indices.get.GetIndexRequest; -import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder; import org.opensearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.index.IndexRequestBuilder; -import org.opensearch.action.support.PlainActionFuture; import org.opensearch.client.Client; -import org.opensearch.client.Requests; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; -import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.rest.RestStatus; -import org.opensearch.index.IndexSettings; import org.opensearch.indices.InvalidIndexNameException; -import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.repositories.RepositoriesService; -import org.opensearch.test.InternalTestCluster; - -import java.io.IOException; import java.nio.file.Path; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Arrays; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -86,8 +71,6 @@ import static org.hamcrest.Matchers.nullValue; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; import static org.opensearch.index.IndexSettings.INDEX_REFRESH_INTERVAL_SETTING; import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.opensearch.index.query.QueryBuilders.matchQuery; @@ -174,494 +157,6 @@ public void testParallelRestoreOperations() { assertThat(client.prepareGet(restoredIndexName2, docId2).get().isExists(), equalTo(true)); } - public void testRestoreRemoteStoreIndicesWithRemoteTranslog() throws IOException, ExecutionException, InterruptedException { - testRestoreOperationsShallowCopyEnabled(); - } - - public void testRestoreOperationsShallowCopyEnabled() throws IOException, ExecutionException, InterruptedException { - String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); - String primary = internalCluster().startDataOnlyNode(); - String indexName1 = "testindex1"; - String indexName2 = "testindex2"; - String snapshotRepoName = "test-restore-snapshot-repo"; - String remoteStoreRepoName = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; - String snapshotName1 = "test-restore-snapshot1"; - String snapshotName2 = "test-restore-snapshot2"; - Path absolutePath1 = randomRepoPath().toAbsolutePath(); - Path absolutePath2 = randomRepoPath().toAbsolutePath(); - logger.info("Snapshot Path [{}]", absolutePath1); - logger.info("Remote Store Repo Path [{}]", absolutePath2); - String restoredIndexName1 = indexName1 + "-restored"; - String restoredIndexName1Seg = indexName1 + "-restored-seg"; - String restoredIndexName1Doc = indexName1 + "-restored-doc"; - String restoredIndexName2 = indexName2 + "-restored"; - - createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true)); - createRepository(remoteStoreRepoName, "fs", absolutePath2); - - Client client = client(); - Settings indexSettings = getIndexSettings(true, remoteStoreRepoName, 1, 0).build(); - createIndex(indexName1, indexSettings); - - Settings indexSettings2 = getIndexSettings(false, null, 1, 0).build(); - createIndex(indexName2, indexSettings2); - - final int numDocsInIndex1 = 5; - final int numDocsInIndex2 = 6; - indexDocuments(client, indexName1, numDocsInIndex1); - indexDocuments(client, indexName2, numDocsInIndex2); - ensureGreen(indexName1, indexName2); - - internalCluster().startDataOnlyNode(); - logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(true) - .setIndices(indexName1, indexName2) - .get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat( - createSnapshotResponse.getSnapshotInfo().successfulShards(), - equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) - ); - assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); - - updateRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); - CreateSnapshotResponse createSnapshotResponse2 = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName2) - .setWaitForCompletion(true) - .setIndices(indexName1, indexName2) - .get(); - assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat( - createSnapshotResponse2.getSnapshotInfo().successfulShards(), - equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()) - ); - assertThat(createSnapshotResponse2.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); - - DeleteResponse deleteResponse = client().prepareDelete(indexName1, "0").execute().actionGet(); - assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); - indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + randomIntBetween(2, 5)); - ensureGreen(indexName1); - - RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() - .cluster() - .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(false) - .setIndices(indexName1) - .setRenamePattern(indexName1) - .setRenameReplacement(restoredIndexName1) - .get(); - RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() - .cluster() - .prepareRestoreSnapshot(snapshotRepoName, snapshotName2) - .setWaitForCompletion(false) - .setIndices(indexName2) - .setRenamePattern(indexName2) - .setRenameReplacement(restoredIndexName2) - .get(); - assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); - assertEquals(restoreSnapshotResponse2.status(), RestStatus.ACCEPTED); - ensureGreen(restoredIndexName1, restoredIndexName2); - assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); - assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2); - - // deleting data for restoredIndexName1 and restoring from remote store. - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); - ensureRed(restoredIndexName1); - // Re-initialize client to make sure we are not using client from stopped node. - client = client(clusterManagerNode); - assertAcked(client.admin().indices().prepareClose(restoredIndexName1)); - client.admin() - .cluster() - .restoreRemoteStore( - new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true), - PlainActionFuture.newFuture() - ); - ensureYellowAndNoInitializingShards(restoredIndexName1); - ensureGreen(restoredIndexName1); - assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1); - // indexing some new docs and validating - indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2); - ensureGreen(restoredIndexName1); - assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); - - // restore index as seg rep enabled with remote store and remote translog disabled - RestoreSnapshotResponse restoreSnapshotResponse3 = client.admin() - .cluster() - .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(false) - .setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED) - .setIndices(indexName1) - .setRenamePattern(indexName1) - .setRenameReplacement(restoredIndexName1Seg) - .get(); - assertEquals(restoreSnapshotResponse3.status(), RestStatus.ACCEPTED); - ensureGreen(restoredIndexName1Seg); - - GetIndexResponse getIndexResponse = client.admin() - .indices() - .getIndex(new GetIndexRequest().indices(restoredIndexName1Seg).includeDefaults(true)) - .get(); - indexSettings = getIndexResponse.settings().get(restoredIndexName1Seg); - assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); - assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null)); - assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE)); - assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1); - // indexing some new docs and validating - indexDocuments(client, restoredIndexName1Seg, numDocsInIndex1, numDocsInIndex1 + 2); - ensureGreen(restoredIndexName1Seg); - assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1 + 2); - - // restore index as doc rep based from shallow copy snapshot - RestoreSnapshotResponse restoreSnapshotResponse4 = client.admin() - .cluster() - .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(false) - .setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, IndexMetadata.SETTING_REPLICATION_TYPE) - .setIndices(indexName1) - .setRenamePattern(indexName1) - .setRenameReplacement(restoredIndexName1Doc) - .get(); - assertEquals(restoreSnapshotResponse4.status(), RestStatus.ACCEPTED); - ensureGreen(restoredIndexName1Doc); - - getIndexResponse = client.admin() - .indices() - .getIndex(new GetIndexRequest().indices(restoredIndexName1Doc).includeDefaults(true)) - .get(); - indexSettings = getIndexResponse.settings().get(restoredIndexName1Doc); - assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); - assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null)); - assertNull(indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE)); - assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1); - // indexing some new docs and validating - indexDocuments(client, restoredIndexName1Doc, numDocsInIndex1, numDocsInIndex1 + 2); - ensureGreen(restoredIndexName1Doc); - assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1 + 2); - } - - public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { - String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); - String primary = internalCluster().startDataOnlyNode(); - String indexName1 = "testindex1"; - String indexName2 = "testindex2"; - String snapshotRepoName = "test-restore-snapshot-repo"; - String remoteStoreRepoName = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; - String snapshotName1 = "test-restore-snapshot1"; - String snapshotName2 = "test-restore-snapshot2"; - Path absolutePath1 = randomRepoPath().toAbsolutePath(); - Path absolutePath2 = randomRepoPath().toAbsolutePath(); - logger.info("Snapshot Path [{}]", absolutePath1); - logger.info("Remote Store Repo Path [{}]", absolutePath2); - String restoredIndexName2 = indexName2 + "-restored"; - - boolean enableShallowCopy = randomBoolean(); - createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, enableShallowCopy)); - createRepository(remoteStoreRepoName, "fs", absolutePath2); - - Client client = client(); - Settings indexSettings = getIndexSettings(true, remoteStoreRepoName, 1, 0).build(); - createIndex(indexName1, indexSettings); - - Settings indexSettings2 = getIndexSettings(false, null, 1, 0).build(); - createIndex(indexName2, indexSettings2); - - final int numDocsInIndex1 = 5; - final int numDocsInIndex2 = 6; - indexDocuments(client, indexName1, numDocsInIndex1); - indexDocuments(client, indexName2, numDocsInIndex2); - ensureGreen(indexName1, indexName2); - - internalCluster().startDataOnlyNode(); - logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(true) - .setIndices(indexName1, indexName2) - .get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat( - createSnapshotResponse.getSnapshotInfo().successfulShards(), - equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) - ); - assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); - - updateRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); - CreateSnapshotResponse createSnapshotResponse2 = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName2) - .setWaitForCompletion(true) - .setIndices(indexName1, indexName2) - .get(); - assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat( - createSnapshotResponse2.getSnapshotInfo().successfulShards(), - equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()) - ); - assertThat(createSnapshotResponse2.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); - - DeleteResponse deleteResponse = client().prepareDelete(indexName1, "0").execute().actionGet(); - assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); - indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + randomIntBetween(2, 5)); - ensureGreen(indexName1); - - assertAcked(client().admin().indices().prepareClose(indexName1)); - - RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() - .cluster() - .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(false) - .setIndices(indexName1) - .get(); - RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() - .cluster() - .prepareRestoreSnapshot(snapshotRepoName, snapshotName2) - .setWaitForCompletion(false) - .setIndices(indexName2) - .setRenamePattern(indexName2) - .setRenameReplacement(restoredIndexName2) - .get(); - assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); - assertEquals(restoreSnapshotResponse2.status(), RestStatus.ACCEPTED); - ensureGreen(indexName1, restoredIndexName2); - assertDocsPresentInIndex(client, indexName1, numDocsInIndex1); - assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2); - - // deleting data for restoredIndexName1 and restoring from remote store. - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); - ensureRed(indexName1); - // Re-initialize client to make sure we are not using client from stopped node. - client = client(clusterManagerNode); - assertAcked(client.admin().indices().prepareClose(indexName1)); - client.admin() - .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1).restoreAllShards(true), PlainActionFuture.newFuture()); - ensureYellowAndNoInitializingShards(indexName1); - ensureGreen(indexName1); - assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); - // indexing some new docs and validating - indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + 2); - ensureGreen(indexName1); - assertDocsPresentInIndex(client, indexName1, numDocsInIndex1 + 2); - } - - public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException { - String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); - String primary = internalCluster().startDataOnlyNode(); - String indexName1 = "testindex1"; - String indexName2 = "testindex2"; - String snapshotRepoName = "test-restore-snapshot-repo"; - String remoteStoreRepoName = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; - String remoteStoreRepo2Name = "test-rs-repo-2" + TEST_REMOTE_STORE_REPO_SUFFIX; - String snapshotName1 = "test-restore-snapshot1"; - Path absolutePath1 = randomRepoPath().toAbsolutePath(); - Path absolutePath2 = randomRepoPath().toAbsolutePath(); - Path absolutePath3 = randomRepoPath().toAbsolutePath(); - String restoredIndexName1 = indexName1 + "-restored"; - - createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); - createRepository(remoteStoreRepoName, "fs", absolutePath2); - createRepository(remoteStoreRepo2Name, "fs", absolutePath3); - - Client client = client(); - Settings indexSettings = getIndexSettings(true, remoteStoreRepoName, 1, 0).build(); - createIndex(indexName1, indexSettings); - - Settings indexSettings2 = getIndexSettings(false, null, 1, 0).build(); - createIndex(indexName2, indexSettings2); - - final int numDocsInIndex1 = 5; - final int numDocsInIndex2 = 6; - indexDocuments(client, indexName1, numDocsInIndex1); - indexDocuments(client, indexName2, numDocsInIndex2); - ensureGreen(indexName1, indexName2); - - internalCluster().startDataOnlyNode(); - - logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(true) - .setIndices(indexName1, indexName2) - .get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat( - createSnapshotResponse.getSnapshotInfo().successfulShards(), - equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) - ); - assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); - - Settings remoteStoreIndexSettings = Settings.builder() - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo2Name) - .build(); - // restore index as a remote store index with different remote store repo - RestoreSnapshotResponse restoreSnapshotResponse = client.admin() - .cluster() - .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(false) - .setIndexSettings(remoteStoreIndexSettings) - .setIndices(indexName1) - .setRenamePattern(indexName1) - .setRenameReplacement(restoredIndexName1) - .get(); - assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED); - ensureGreen(restoredIndexName1); - assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1); - - // deleting data for restoredIndexName1 and restoring from remote store. - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); - // Re-initialize client to make sure we are not using client from stopped node. - client = client(clusterManagerNode); - assertAcked(client.admin().indices().prepareClose(restoredIndexName1)); - client.admin() - .cluster() - .restoreRemoteStore( - new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true), - PlainActionFuture.newFuture() - ); - ensureYellowAndNoInitializingShards(restoredIndexName1); - ensureGreen(restoredIndexName1); - // indexing some new docs and validating - assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); - indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2); - ensureGreen(restoredIndexName1); - assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); - } - - private Settings.Builder getIndexSettings(boolean enableRemoteStore, String remoteStoreRepo, int numOfShards, int numOfReplicas) { - Settings.Builder settingsBuilder = Settings.builder() - .put(super.indexSettings()) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas); - if (enableRemoteStore) { - settingsBuilder.put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStoreRepo) - .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); - } - return settingsBuilder; - } - - public void testRestoreShallowSnapshotRepositoryOverriden() throws ExecutionException, InterruptedException { - String indexName1 = "testindex1"; - String snapshotRepoName = "test-restore-snapshot-repo"; - String remoteStoreRepoName = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; - String remoteStoreRepoNameUpdated = "test-rs-repo-updated" + TEST_REMOTE_STORE_REPO_SUFFIX; - String snapshotName1 = "test-restore-snapshot1"; - Path absolutePath1 = randomRepoPath().toAbsolutePath(); - Path absolutePath2 = randomRepoPath().toAbsolutePath(); - Path absolutePath3 = randomRepoPath().toAbsolutePath(); - String[] pathTokens = absolutePath1.toString().split("/"); - String basePath = pathTokens[pathTokens.length - 1]; - Arrays.copyOf(pathTokens, pathTokens.length - 1); - Path location = PathUtils.get(String.join("/", pathTokens)); - pathTokens = absolutePath2.toString().split("/"); - String basePath2 = pathTokens[pathTokens.length - 1]; - Arrays.copyOf(pathTokens, pathTokens.length - 1); - Path location2 = PathUtils.get(String.join("/", pathTokens)); - logger.info("Path 1 [{}]", absolutePath1); - logger.info("Path 2 [{}]", absolutePath2); - logger.info("Path 3 [{}]", absolutePath3); - String restoredIndexName1 = indexName1 + "-restored"; - - createRepository(snapshotRepoName, "fs", getRepositorySettings(location, basePath, true)); - createRepository(remoteStoreRepoName, "fs", absolutePath3); - - Client client = client(); - Settings indexSettings = Settings.builder() - .put(super.indexSettings()) - .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepoName) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStoreRepoName) - .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - createIndex(indexName1, indexSettings); - - int numDocsInIndex1 = randomIntBetween(2, 5); - indexDocuments(client, indexName1, numDocsInIndex1); - - ensureGreen(indexName1); - - logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(true) - .setIndices(indexName1) - .get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat( - createSnapshotResponse.getSnapshotInfo().successfulShards(), - equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()) - ); - assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS)); - - createRepository(remoteStoreRepoName, "fs", absolutePath2); - - RestoreSnapshotResponse restoreSnapshotResponse = client.admin() - .cluster() - .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(true) - .setIndices(indexName1) - .setRenamePattern(indexName1) - .setRenameReplacement(restoredIndexName1) - .get(); - - assertTrue(restoreSnapshotResponse.getRestoreInfo().failedShards() > 0); - - ensureRed(restoredIndexName1); - - client().admin().indices().close(Requests.closeIndexRequest(restoredIndexName1)).get(); - createRepository(remoteStoreRepoNameUpdated, "fs", absolutePath3); - RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() - .cluster() - .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) - .setWaitForCompletion(true) - .setIndices(indexName1) - .setRenamePattern(indexName1) - .setRenameReplacement(restoredIndexName1) - .setSourceRemoteStoreRepository(remoteStoreRepoNameUpdated) - .get(); - - assertTrue(restoreSnapshotResponse2.getRestoreInfo().failedShards() == 0); - ensureGreen(restoredIndexName1); - assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); - - // indexing some new docs and validating - indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2); - ensureGreen(restoredIndexName1); - assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); - } - - private void indexDocuments(Client client, String indexName, int numOfDocs) { - indexDocuments(client, indexName, 0, numOfDocs); - } - - private void indexDocuments(Client client, String indexName, int fromId, int toId) { - for (int i = fromId; i < toId; i++) { - String id = Integer.toString(i); - client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); - } - client.admin().indices().prepareFlush(indexName).get(); - } - - private void assertDocsPresentInIndex(Client client, String indexName, int numOfDocs) { - for (int i = 0; i < numOfDocs; i++) { - String id = Integer.toString(i); - logger.info("checking for index " + indexName + " with docId" + id); - assertTrue("doc with id" + id + " is not present for index " + indexName, client.prepareGet(indexName, id).get().isExists()); - } - } - public void testParallelRestoreOperationsFromSingleSnapshot() throws Exception { String indexName1 = "testindex1"; String indexName2 = "testindex2"; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java index c22dd90cc930b..33bfc79523752 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java @@ -112,7 +112,7 @@ public void testStatusApiConsistency() { assertEquals(snStatus.getStats().getTime(), snapshotInfo.endTime() - snapshotInfo.startTime()); } - public void testStatusAPICallForShallowCopySnapshot() throws Exception { + public void testStatusAPICallForShallowCopySnapshot() { disableRepoConsistencyCheck("Remote store repository is being used for the test"); internalCluster().startClusterManagerOnlyNode(); internalCluster().startDataOnlyNode(); @@ -120,10 +120,6 @@ public void testStatusAPICallForShallowCopySnapshot() throws Exception { final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); - final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; createIndex(indexName); ensureGreen(); @@ -133,20 +129,8 @@ public void testStatusAPICallForShallowCopySnapshot() throws Exception { } refresh(); - final String remoteStoreEnabledIndexName = "remote-index-1"; - final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); - createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index(remoteStoreEnabledIndexName, "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - final String snapshot = "snapshot"; createFullSnapshot(snapshotRepoName, snapshot); - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); final SnapshotStatus snapshotStatus = getSnapshotStatus(snapshotRepoName, snapshot); assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); @@ -157,14 +141,6 @@ public void testStatusAPICallForShallowCopySnapshot() throws Exception { assertThat(snapshotShardState.getStats().getTotalSize(), greaterThan(0L)); assertThat(snapshotShardState.getStats().getIncrementalFileCount(), greaterThan(0)); assertThat(snapshotShardState.getStats().getIncrementalSize(), greaterThan(0L)); - - // Validating that the incremental file count and incremental file size is zero for shallow copy - final SnapshotIndexShardStatus shallowSnapshotShardState = stateFirstShard(snapshotStatus, remoteStoreEnabledIndexName); - assertThat(shallowSnapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE)); - assertThat(shallowSnapshotShardState.getStats().getTotalFileCount(), greaterThan(0)); - assertThat(shallowSnapshotShardState.getStats().getTotalSize(), greaterThan(0L)); - assertThat(shallowSnapshotShardState.getStats().getIncrementalFileCount(), is(0)); - assertThat(shallowSnapshotShardState.getStats().getIncrementalSize(), is(0L)); } public void testStatusAPICallInProgressSnapshot() throws Exception { @@ -245,63 +221,6 @@ public void testExceptionOnMissingShardLevelSnapBlob() throws IOException { ); } - public void testStatusAPIStatsForBackToBackShallowSnapshot() throws Exception { - disableRepoConsistencyCheck("Remote store repository is being used for the test"); - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNode(); - - final String snapshotRepoName = "snapshot-repo-name"; - createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); - - final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - - final String indexName = "index-1"; - createIndex(indexName); - ensureGreen(); - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - - final String remoteStoreEnabledIndexName = "remote-index-1"; - final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); - createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index(remoteStoreEnabledIndexName, "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - - createFullSnapshot(snapshotRepoName, "test-snap-1"); - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); - - SnapshotStatus snapshotStatus = getSnapshotStatus(snapshotRepoName, "test-snap-1"); - assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); - - SnapshotIndexShardStatus shallowSnapshotShardState = stateFirstShard(snapshotStatus, remoteStoreEnabledIndexName); - assertThat(shallowSnapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE)); - final int totalFileCount = shallowSnapshotShardState.getStats().getTotalFileCount(); - final long totalSize = shallowSnapshotShardState.getStats().getTotalSize(); - final int incrementalFileCount = shallowSnapshotShardState.getStats().getIncrementalFileCount(); - final long incrementalSize = shallowSnapshotShardState.getStats().getIncrementalSize(); - - createFullSnapshot(snapshotRepoName, "test-snap-2"); - assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 2); - - snapshotStatus = getSnapshotStatus(snapshotRepoName, "test-snap-2"); - assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); - shallowSnapshotShardState = stateFirstShard(snapshotStatus, remoteStoreEnabledIndexName); - assertThat(shallowSnapshotShardState.getStats().getTotalFileCount(), equalTo(totalFileCount)); - assertThat(shallowSnapshotShardState.getStats().getTotalSize(), equalTo(totalSize)); - assertThat(shallowSnapshotShardState.getStats().getIncrementalFileCount(), equalTo(incrementalFileCount)); - assertThat(shallowSnapshotShardState.getStats().getIncrementalSize(), equalTo(incrementalSize)); - } - public void testGetSnapshotsWithoutIndices() throws Exception { createRepository("test-repo", "fs"); @@ -441,17 +360,12 @@ public void testSnapshotStatusOnFailedSnapshot() throws Exception { } public void testStatusAPICallInProgressShallowSnapshot() throws Exception { - disableRepoConsistencyCheck("Remote store repository is being used for the test"); internalCluster().startClusterManagerOnlyNode(); internalCluster().startDataOnlyNode(); final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy().put("block_on_data", true)); - final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "mock", remoteStoreRepoPath); - final String indexName = "index-1"; createIndex(indexName); ensureGreen(); @@ -461,17 +375,6 @@ public void testStatusAPICallInProgressShallowSnapshot() throws Exception { } refresh(); - final String remoteStoreEnabledIndexName = "remote-index-1"; - final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName); - createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index(remoteStoreEnabledIndexName, "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - logger.info("--> snapshot"); ActionFuture createSnapshotResponseActionFuture = startFullSnapshot(snapshotRepoName, "test-snap"); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 294e0d7e06bd3..4d8b70f602c50 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -133,7 +133,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_REPOSITORY_SETTING; +import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; @@ -966,7 +966,7 @@ private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, if (isRemoteStoreClusterEnabled == true) { settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.get(clusterSettings)) + .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(clusterSettings)) .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(clusterSettings)); } } @@ -974,8 +974,8 @@ private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, private static List getRemoteStoreOverriddenSetting(Settings requestSettings) { return Stream.of( INDEX_REMOTE_STORE_ENABLED_SETTING, - INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING, - INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING + INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING, + INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING ).filter(setting -> setting.exists(requestSettings)).map(Setting::getKey).collect(toList()); } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 46a43842451d9..4c720e5f7afed 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -671,7 +671,7 @@ public void apply(Settings value, Settings current, Settings previous) { List.of(FeatureFlags.REMOTE_STORE), List.of( IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING, - IndicesService.CLUSTER_REMOTE_STORE_REPOSITORY_SETTING, + IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING, IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING ), List.of(FeatureFlags.CONCURRENT_SEGMENT_SEARCH), diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index b574ffd1006c0..dc3237e4e7eac 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -256,7 +256,7 @@ public class IndicesService extends AbstractLifecycleComponent /** * Used to specify default repo to use for segment upload for remote store backed indices */ - public static final Setting CLUSTER_REMOTE_STORE_REPOSITORY_SETTING = Setting.simpleString( + public static final Setting CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING = Setting.simpleString( "cluster.remote_store.segment.repository", "", Property.NodeScope, diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index d7e89172c5837..9b32a97aa5c6c 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -227,7 +227,12 @@ public ClusterState execute(ClusterState currentState) { List indicesToBeRestored = new ArrayList<>(); int totalShards = 0; - for (String index : request.indices()) { + List filteredIndices = filterIndices( + Arrays.asList(request.indices()), + currentState.metadata().indices().keySet().toArray(new String[0]), + IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN + ); + for (String index : filteredIndices) { IndexMetadata currentIndexMetadata = currentState.metadata().index(index); if (currentIndexMetadata == null) { // ToDo: Handle index metadata does not exist case. (GitHub #3457) diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 5384bf942f924..2b80c2a139e2c 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -144,7 +144,7 @@ import static org.opensearch.cluster.metadata.MetadataCreateIndexService.resolveAndValidateAliases; import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_REPOSITORY_SETTING; +import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; @@ -1196,7 +1196,7 @@ public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettin Settings settings = Settings.builder() .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT) .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") + .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -1229,7 +1229,7 @@ public void testRemoteStoreNoUserOverrideIndexSettings() { Settings settings = Settings.builder() .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") + .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -1259,7 +1259,7 @@ public void testRemoteStoreDisabledByUserIndexSettings() { Settings settings = Settings.builder() .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") + .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -1298,7 +1298,7 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { Settings settings = Settings.builder() .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") + .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -1340,7 +1340,7 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() { Settings settings = Settings.builder() .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") + .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java new file mode 100644 index 0000000000000..0dbc0372458b5 --- /dev/null +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java @@ -0,0 +1,140 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.blobstore; + +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.index.IndexModule; +import org.opensearch.index.IndexService; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.store.RemoteBufferedOutputDirectory; +import org.opensearch.indices.IndicesService; +import org.opensearch.indices.recovery.RecoverySettings; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.RepositoryPlugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.Repository; +import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.OpenSearchSingleNodeTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class BlobStoreRepositoryHelperTests extends OpenSearchSingleNodeTestCase { + + static final String REPO_TYPE = "fsLike"; + + protected Collection> getPlugins() { + return Arrays.asList(FsLikeRepoPlugin.class); + } + + protected String[] getLockFilesInRemoteStore(String remoteStoreIndex, String remoteStoreRepository) throws IOException { + String indexUUID = client().admin() + .indices() + .prepareGetSettings(remoteStoreIndex) + .get() + .getSetting(remoteStoreIndex, IndexMetadata.SETTING_INDEX_UUID); + final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); + final BlobStoreRepository remoteStorerepository = (BlobStoreRepository) repositoriesService.repository(remoteStoreRepository); + BlobPath shardLevelBlobPath = remoteStorerepository.basePath().add(indexUUID).add("0").add("segments").add("lock_files"); + BlobContainer blobContainer = remoteStorerepository.blobStore().blobContainer(shardLevelBlobPath); + try (RemoteBufferedOutputDirectory lockDirectory = new RemoteBufferedOutputDirectory(blobContainer)) { + return Arrays.stream(lockDirectory.listAll()).filter(lock -> lock.endsWith(".lock")).toArray(String[]::new); + } + } + + // the reason for this plug-in is to drop any assertSnapshotOrGenericThread as mostly all access in this test goes from test threads + public static class FsLikeRepoPlugin extends Plugin implements RepositoryPlugin { + + @Override + public Map getRepositories( + Environment env, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + RecoverySettings recoverySettings + ) { + return Collections.singletonMap( + REPO_TYPE, + (metadata) -> new FsRepository(metadata, env, namedXContentRegistry, clusterService, recoverySettings) { + @Override + protected void assertSnapshotOrGenericThread() { + // eliminate thread name check as we access blobStore on test/main threads + } + } + ); + } + } + + protected void createRepository(Client client, String repoName) { + AcknowledgedResponse putRepositoryResponse = client.admin() + .cluster() + .preparePutRepository(repoName) + .setType(REPO_TYPE) + .setSettings( + Settings.builder().put(node().settings()).put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) + ) + .get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + } + + protected void createRepository(Client client, String repoName, Settings repoSettings) { + AcknowledgedResponse putRepositoryResponse = client.admin() + .cluster() + .preparePutRepository(repoName) + .setType(REPO_TYPE) + .setSettings(repoSettings) + .get(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + } + + protected void updateRepository(Client client, String repoName, Settings repoSettings) { + createRepository(client, repoName, repoSettings); + } + + protected Settings getRemoteStoreBackedIndexSettings(String remoteStoreRepo) { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "1") + .put("index.refresh_interval", "300s") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1") + .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey()) + .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(); + } + + protected void indexDocuments(Client client, String indexName) { + int numDocs = randomIntBetween(10, 20); + for (int i = 0; i < numDocs; i++) { + String id = Integer.toString(i); + client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); + } + client.admin().indices().prepareFlush(indexName).get(); + } + + protected IndexSettings getIndexSettings(String indexName) { + final IndicesService indicesService = getInstanceFromNode(IndicesService.class); + final IndexService indexService = indicesService.indexService(resolveIndex(indexName)); + return indexService.getIndexSettings(); + } + +} diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java new file mode 100644 index 0000000000000..e8b1f559ac1b8 --- /dev/null +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -0,0 +1,373 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.repositories.blobstore; + +import org.apache.lucene.tests.util.LuceneTestCase; +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; +import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.repositories.IndexId; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.RepositoryData; +import org.opensearch.snapshots.SnapshotId; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; +import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; +import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; +import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; + +/** + * Tests for the {@link BlobStoreRepository} and its subclasses. + */ +@LuceneTestCase.SuppressFileSystems("ExtrasFS") +public class BlobStoreRepositoryRemoteIndexTests extends BlobStoreRepositoryHelperTests { + + @Override + protected Settings nodeSettings() { + return Settings.builder() + .put(super.nodeSettings()) + .put(FeatureFlags.REMOTE_STORE, "true") + .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) + .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) + .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "test-rs-repo") + .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "test-rs-repo") + .build(); + } + + // Validate Scenario Normal Snapshot -> remoteStoreShallowCopy Snapshot -> normal Snapshot + public void testRetrieveShallowCopySnapshotCase1() throws IOException { + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + final Client client = client(); + final String snapshotRepositoryName = "test-repo"; + final String remoteStoreRepositoryName = "test-rs-repo"; + + logger.info("--> creating snapshot repository"); + + Settings snapshotRepoSettings = Settings.builder() + .put(node().settings()) + .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) + .build(); + createRepository(client, snapshotRepositoryName, snapshotRepoSettings); + + logger.info("--> creating remote store repository"); + Settings remoteStoreRepoSettings = Settings.builder() + .put(node().settings()) + .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) + .build(); + createRepository(client, remoteStoreRepositoryName, remoteStoreRepoSettings); + + logger.info("--> creating an index and indexing documents"); + final String indexName = "test-idx"; + createIndex(indexName); + ensureGreen(); + indexDocuments(client, indexName); + + logger.info("--> creating a remote store enabled index and indexing documents"); + final String remoteStoreIndexName = "test-rs-idx"; + Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); + createIndex(remoteStoreIndexName, indexSettings); + indexDocuments(client, remoteStoreIndexName); + + logger.info("--> create first snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-1") + .setWaitForCompletion(true) + .setIndices(indexName, remoteStoreIndexName) + .get(); + final SnapshotId snapshotId1 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + + String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); + assert (lockFiles.length == 0) : "there should be no lock files present in directory, but found " + Arrays.toString(lockFiles); + logger.info("--> create remote index shallow snapshot"); + Settings snapshotRepoSettingsForShallowCopy = Settings.builder() + .put(snapshotRepoSettings) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), Boolean.TRUE) + .build(); + updateRepository(client, snapshotRepositoryName, snapshotRepoSettingsForShallowCopy); + + createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-2") + .setWaitForCompletion(true) + .setIndices(indexName, remoteStoreIndexName) + .get(); + final SnapshotId snapshotId2 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + + lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); + assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); + assert lockFiles[0].endsWith(snapshotId2.getUUID() + ".lock"); + + logger.info("--> create another normal snapshot"); + updateRepository(client, snapshotRepositoryName, snapshotRepoSettings); + createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-3") + .setWaitForCompletion(true) + .setIndices(indexName, remoteStoreIndexName) + .get(); + final SnapshotId snapshotId3 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + + lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); + assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); + assert lockFiles[0].endsWith(snapshotId2.getUUID() + ".lock"); + + logger.info("--> make sure the node's repository can resolve the snapshots"); + final List originalSnapshots = Arrays.asList(snapshotId1, snapshotId2, snapshotId3); + + final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); + final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(snapshotRepositoryName); + RepositoryData repositoryData = OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository); + IndexId indexId = repositoryData.resolveIndexId(remoteStoreIndexName); + + List snapshotIds = repositoryData.getSnapshotIds() + .stream() + .sorted((s1, s2) -> s1.getName().compareTo(s2.getName())) + .collect(Collectors.toList()); + assertThat(snapshotIds, equalTo(originalSnapshots)); + + // shallow copy shard metadata - getRemoteStoreShallowCopyShardMetadata + RemoteStoreShardShallowCopySnapshot shardShallowCopySnapshot = repository.getRemoteStoreShallowCopyShardMetadata( + snapshotId2, + indexId, + new ShardId(remoteStoreIndexName, indexId.getId(), 0) + ); + assertEquals(shardShallowCopySnapshot.getRemoteStoreRepository(), remoteStoreRepositoryName); + } + + public void testGetRemoteStoreShallowCopyShardMetadata() throws IOException { + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + final Client client = client(); + final String snapshotRepositoryName = "test-repo"; + final String remoteStoreRepositoryName = "test-rs-repo"; + + logger.info("--> creating snapshot repository"); + + Settings snapshotRepoSettings = Settings.builder() + .put(node().settings()) + .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) + .build(); + createRepository(client, snapshotRepositoryName, snapshotRepoSettings); + + logger.info("--> creating remote store repository"); + Settings remoteStoreRepoSettings = Settings.builder() + .put(node().settings()) + .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) + .build(); + createRepository(client, remoteStoreRepositoryName, remoteStoreRepoSettings); + + logger.info("--> creating a remote store enabled index and indexing documents"); + final String remoteStoreIndexName = "test-rs-idx"; + Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); + createIndex(remoteStoreIndexName, indexSettings); + indexDocuments(client, remoteStoreIndexName); + + logger.info("--> create remote index shallow snapshot"); + Settings snapshotRepoSettingsForShallowCopy = Settings.builder() + .put(snapshotRepoSettings) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), Boolean.TRUE) + .build(); + updateRepository(client, snapshotRepositoryName, snapshotRepoSettingsForShallowCopy); + + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-2") + .setWaitForCompletion(true) + .setIndices(remoteStoreIndexName) + .get(); + final SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId(); + + String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); + assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); + assert lockFiles[0].endsWith(snapshotId.getUUID() + ".lock"); + + final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); + final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(snapshotRepositoryName); + RepositoryData repositoryData = OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository); + IndexSettings indexSetting = getIndexSettings(remoteStoreIndexName); + IndexId indexId = repositoryData.resolveIndexId(remoteStoreIndexName); + RemoteStoreShardShallowCopySnapshot shardShallowCopySnapshot = repository.getRemoteStoreShallowCopyShardMetadata( + snapshotId, + indexId, + new ShardId(remoteStoreIndexName, indexSetting.getUUID(), 0) + ); + assertEquals(shardShallowCopySnapshot.getRemoteStoreRepository(), remoteStoreRepositoryName); + assertEquals(shardShallowCopySnapshot.getIndexUUID(), indexSetting.getUUID()); + assertEquals(shardShallowCopySnapshot.getRepositoryBasePath(), ""); + } + + // Validate Scenario remoteStoreShallowCopy Snapshot -> remoteStoreShallowCopy Snapshot + // -> remoteStoreShallowCopy Snapshot -> normal snapshot + public void testRetrieveShallowCopySnapshotCase2() throws IOException { + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + final Client client = client(); + final String snapshotRepositoryName = "test-repo"; + final String remoteStoreRepositoryName = "test-rs-repo"; + + logger.info("--> creating snapshot repository"); + Settings snapshotRepoSettings = Settings.builder() + .put(node().settings()) + .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) + .build(); + createRepository(client, snapshotRepositoryName, snapshotRepoSettings); + + GetRepositoriesResponse updatedGetRepositoriesResponse = client.admin() + .cluster() + .prepareGetRepositories(snapshotRepositoryName) + .get(); + + RepositoryMetadata updatedRepositoryMetadata = updatedGetRepositoriesResponse.repositories().get(0); + + assertFalse(updatedRepositoryMetadata.settings().getAsBoolean(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), false)); + + logger.info("--> creating remote store repository"); + createRepository(client, remoteStoreRepositoryName); + + logger.info("--> creating an index and indexing documents"); + final String indexName = "test-idx"; + createIndex(indexName); + ensureGreen(); + indexDocuments(client, indexName); + + logger.info("--> creating a remote store enabled index and indexing documents"); + final String remoteStoreIndexName = "test-rs-idx"; + Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); + createIndex(remoteStoreIndexName, indexSettings); + indexDocuments(client, remoteStoreIndexName); + + logger.info("--> create first remote index shallow snapshot"); + + Settings snapshotRepoSettingsForShallowCopy = Settings.builder() + .put(snapshotRepoSettings) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .build(); + updateRepository(client, snapshotRepositoryName, snapshotRepoSettingsForShallowCopy); + + updatedGetRepositoriesResponse = client.admin().cluster().prepareGetRepositories(snapshotRepositoryName).get(); + + updatedRepositoryMetadata = updatedGetRepositoriesResponse.repositories().get(0); + + assertTrue(updatedRepositoryMetadata.settings().getAsBoolean(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), false)); + + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-1") + .setWaitForCompletion(true) + .setIndices(indexName, remoteStoreIndexName) + .get(); + final SnapshotId snapshotId1 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + + String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); + assert (lockFiles.length == 1) : "lock files are " + Arrays.toString(lockFiles); + assert lockFiles[0].endsWith(snapshotId1.getUUID() + ".lock"); + + logger.info("--> create second remote index shallow snapshot"); + createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-2") + .setWaitForCompletion(true) + .setIndices(indexName, remoteStoreIndexName) + .get(); + final SnapshotId snapshotId2 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + + lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); + assert (lockFiles.length == 2) : "lock files are " + Arrays.toString(lockFiles); + List shallowCopySnapshotIDs = Arrays.asList(snapshotId1, snapshotId2); + for (SnapshotId snapshotId : shallowCopySnapshotIDs) { + assert lockFiles[0].contains(snapshotId.getUUID()) || lockFiles[1].contains(snapshotId.getUUID()); + } + logger.info("--> create third remote index shallow snapshot"); + createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-3") + .setWaitForCompletion(true) + .setIndices(indexName, remoteStoreIndexName) + .get(); + final SnapshotId snapshotId3 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + + lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); + assert (lockFiles.length == 3); + shallowCopySnapshotIDs = Arrays.asList(snapshotId1, snapshotId2, snapshotId3); + for (SnapshotId snapshotId : shallowCopySnapshotIDs) { + assert lockFiles[0].contains(snapshotId.getUUID()) + || lockFiles[1].contains(snapshotId.getUUID()) + || lockFiles[2].contains(snapshotId.getUUID()); + } + logger.info("--> create normal snapshot"); + createRepository(client, snapshotRepositoryName, snapshotRepoSettings); + createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-4") + .setWaitForCompletion(true) + .setIndices(indexName, remoteStoreIndexName) + .get(); + final SnapshotId snapshotId4 = createSnapshotResponse.getSnapshotInfo().snapshotId(); + + lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); + assert (lockFiles.length == 3) : "lock files are " + Arrays.toString(lockFiles); + shallowCopySnapshotIDs = Arrays.asList(snapshotId1, snapshotId2, snapshotId3); + for (SnapshotId snapshotId : shallowCopySnapshotIDs) { + assert lockFiles[0].contains(snapshotId.getUUID()) + || lockFiles[1].contains(snapshotId.getUUID()) + || lockFiles[2].contains(snapshotId.getUUID()); + } + + logger.info("--> make sure the node's repository can resolve the snapshots"); + final List originalSnapshots = Arrays.asList(snapshotId1, snapshotId2, snapshotId3, snapshotId4); + + final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); + final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(snapshotRepositoryName); + List snapshotIds = OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository) + .getSnapshotIds() + .stream() + .sorted((s1, s2) -> s1.getName().compareTo(s2.getName())) + .collect(Collectors.toList()); + assertThat(snapshotIds, equalTo(originalSnapshots)); + } + +} diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index 28513f279f8ad..bf712f80305b7 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -34,50 +34,27 @@ import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; -import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; -import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; -import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.util.FeatureFlags; -import org.opensearch.core.index.shard.ShardId; -import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; -import org.opensearch.index.IndexModule; -import org.opensearch.index.IndexService; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; -import org.opensearch.index.store.RemoteBufferedOutputDirectory; -import org.opensearch.indices.IndicesService; -import org.opensearch.indices.recovery.RecoverySettings; -import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.plugins.Plugin; -import org.opensearch.plugins.RepositoryPlugin; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; -import org.opensearch.repositories.Repository; import org.opensearch.repositories.RepositoryData; import org.opensearch.repositories.RepositoryException; import org.opensearch.repositories.ShardGenerations; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotState; -import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.OpenSearchIntegTestCase; -import org.opensearch.test.OpenSearchSingleNodeTestCase; -import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -92,40 +69,7 @@ * Tests for the {@link BlobStoreRepository} and its subclasses. */ @LuceneTestCase.SuppressFileSystems("ExtrasFS") -public class BlobStoreRepositoryTests extends OpenSearchSingleNodeTestCase { - - static final String REPO_TYPE = "fsLike"; - - protected Collection> getPlugins() { - return Arrays.asList(FsLikeRepoPlugin.class); - } - - // the reason for this plug-in is to drop any assertSnapshotOrGenericThread as mostly all access in this test goes from test threads - public static class FsLikeRepoPlugin extends Plugin implements RepositoryPlugin { - - @Override - public Map getRepositories( - Environment env, - NamedXContentRegistry namedXContentRegistry, - ClusterService clusterService, - RecoverySettings recoverySettings - ) { - return Collections.singletonMap( - REPO_TYPE, - (metadata) -> new FsRepository(metadata, env, namedXContentRegistry, clusterService, recoverySettings) { - @Override - protected void assertSnapshotOrGenericThread() { - // eliminate thread name check as we access blobStore on test/main threads - } - } - ); - } - } - - @Override - protected Settings nodeSettings() { - return Settings.builder().put(super.nodeSettings()).put(FeatureFlags.REMOTE_STORE, "true").build(); - } +public class BlobStoreRepositoryTests extends BlobStoreRepositoryHelperTests { public void testRetrieveSnapshots() throws Exception { final Client client = client(); @@ -183,367 +127,8 @@ public void testRetrieveSnapshots() throws Exception { assertThat(snapshotIds, equalTo(originalSnapshots)); } - private void createRepository(Client client, String repoName) { - AcknowledgedResponse putRepositoryResponse = client.admin() - .cluster() - .preparePutRepository(repoName) - .setType(REPO_TYPE) - .setSettings( - Settings.builder().put(node().settings()).put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) - ) - .get(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - } - - private void createRepository(Client client, String repoName, Settings repoSettings) { - AcknowledgedResponse putRepositoryResponse = client.admin() - .cluster() - .preparePutRepository(repoName) - .setType(REPO_TYPE) - .setSettings(repoSettings) - .get(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - } - - private void updateRepository(Client client, String repoName, Settings repoSettings) { - createRepository(client, repoName, repoSettings); - } - - private Settings getRemoteStoreBackedIndexSettings(String remoteStoreRepo) { - return Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "1") - .put("index.refresh_interval", "300s") - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1") - .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey()) - .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStoreRepo) - .build(); - } - - private void indexDocuments(Client client, String indexName) { - int numDocs = randomIntBetween(10, 20); - for (int i = 0; i < numDocs; i++) { - String id = Integer.toString(i); - client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); - } - client.admin().indices().prepareFlush(indexName).get(); - } - - private String[] getLockFilesInRemoteStore(String remoteStoreIndex, String remoteStoreRepository) throws IOException { - String indexUUID = client().admin() - .indices() - .prepareGetSettings(remoteStoreIndex) - .get() - .getSetting(remoteStoreIndex, IndexMetadata.SETTING_INDEX_UUID); - final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); - final BlobStoreRepository remoteStorerepository = (BlobStoreRepository) repositoriesService.repository(remoteStoreRepository); - BlobPath shardLevelBlobPath = remoteStorerepository.basePath().add(indexUUID).add("0").add("segments").add("lock_files"); - BlobContainer blobContainer = remoteStorerepository.blobStore().blobContainer(shardLevelBlobPath); - try (RemoteBufferedOutputDirectory lockDirectory = new RemoteBufferedOutputDirectory(blobContainer)) { - return Arrays.stream(lockDirectory.listAll()).filter(lock -> lock.endsWith(".lock")).toArray(String[]::new); - } - } - - // Validate Scenario Normal Snapshot -> remoteStoreShallowCopy Snapshot -> normal Snapshot - public void testRetrieveShallowCopySnapshotCase1() throws IOException { - FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - final Client client = client(); - final String snapshotRepositoryName = "test-repo"; - final String remoteStoreRepositoryName = "test-rs-repo"; - - logger.info("--> creating snapshot repository"); - - Settings snapshotRepoSettings = Settings.builder() - .put(node().settings()) - .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) - .build(); - createRepository(client, snapshotRepositoryName, snapshotRepoSettings); - - logger.info("--> creating remote store repository"); - Settings remoteStoreRepoSettings = Settings.builder() - .put(node().settings()) - .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) - .build(); - createRepository(client, remoteStoreRepositoryName, remoteStoreRepoSettings); - - logger.info("--> creating an index and indexing documents"); - final String indexName = "test-idx"; - createIndex(indexName); - ensureGreen(); - indexDocuments(client, indexName); - - logger.info("--> creating a remote store enabled index and indexing documents"); - final String remoteStoreIndexName = "test-rs-idx"; - Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); - createIndex(remoteStoreIndexName, indexSettings); - indexDocuments(client, remoteStoreIndexName); - - logger.info("--> create first snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-1") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId1 = createSnapshotResponse.getSnapshotInfo().snapshotId(); - - String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 0) : "there should be no lock files present in directory, but found " + Arrays.toString(lockFiles); - logger.info("--> create remote index shallow snapshot"); - Settings snapshotRepoSettingsForShallowCopy = Settings.builder() - .put(snapshotRepoSettings) - .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), Boolean.TRUE) - .build(); - updateRepository(client, snapshotRepositoryName, snapshotRepoSettingsForShallowCopy); - - createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-2") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId2 = createSnapshotResponse.getSnapshotInfo().snapshotId(); - - lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); - assert lockFiles[0].endsWith(snapshotId2.getUUID() + ".lock"); - - logger.info("--> create another normal snapshot"); - updateRepository(client, snapshotRepositoryName, snapshotRepoSettings); - createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-3") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId3 = createSnapshotResponse.getSnapshotInfo().snapshotId(); - - lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); - assert lockFiles[0].endsWith(snapshotId2.getUUID() + ".lock"); - - logger.info("--> make sure the node's repository can resolve the snapshots"); - final List originalSnapshots = Arrays.asList(snapshotId1, snapshotId2, snapshotId3); - - final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); - final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(snapshotRepositoryName); - RepositoryData repositoryData = OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository); - IndexId indexId = repositoryData.resolveIndexId(remoteStoreIndexName); - - List snapshotIds = repositoryData.getSnapshotIds() - .stream() - .sorted((s1, s2) -> s1.getName().compareTo(s2.getName())) - .collect(Collectors.toList()); - assertThat(snapshotIds, equalTo(originalSnapshots)); - - // shallow copy shard metadata - getRemoteStoreShallowCopyShardMetadata - RemoteStoreShardShallowCopySnapshot shardShallowCopySnapshot = repository.getRemoteStoreShallowCopyShardMetadata( - snapshotId2, - indexId, - new ShardId(remoteStoreIndexName, indexId.getId(), 0) - ); - assertEquals(shardShallowCopySnapshot.getRemoteStoreRepository(), remoteStoreRepositoryName); - } - - public void testGetRemoteStoreShallowCopyShardMetadata() throws IOException { - FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - final Client client = client(); - final String snapshotRepositoryName = "test-repo"; - final String remoteStoreRepositoryName = "test-rs-repo"; - - logger.info("--> creating snapshot repository"); - - Settings snapshotRepoSettings = Settings.builder() - .put(node().settings()) - .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) - .build(); - createRepository(client, snapshotRepositoryName, snapshotRepoSettings); - - logger.info("--> creating remote store repository"); - Settings remoteStoreRepoSettings = Settings.builder() - .put(node().settings()) - .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) - .build(); - createRepository(client, remoteStoreRepositoryName, remoteStoreRepoSettings); - - logger.info("--> creating a remote store enabled index and indexing documents"); - final String remoteStoreIndexName = "test-rs-idx"; - Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); - createIndex(remoteStoreIndexName, indexSettings); - indexDocuments(client, remoteStoreIndexName); - - logger.info("--> create remote index shallow snapshot"); - Settings snapshotRepoSettingsForShallowCopy = Settings.builder() - .put(snapshotRepoSettings) - .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), Boolean.TRUE) - .build(); - updateRepository(client, snapshotRepositoryName, snapshotRepoSettingsForShallowCopy); - - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-2") - .setWaitForCompletion(true) - .setIndices(remoteStoreIndexName) - .get(); - final SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId(); - - String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 1) : "there should be only one lock file, but found " + Arrays.toString(lockFiles); - assert lockFiles[0].endsWith(snapshotId.getUUID() + ".lock"); - - final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); - final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(snapshotRepositoryName); - RepositoryData repositoryData = OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository); - IndexSettings indexSetting = getIndexSettings(remoteStoreIndexName); - IndexId indexId = repositoryData.resolveIndexId(remoteStoreIndexName); - RemoteStoreShardShallowCopySnapshot shardShallowCopySnapshot = repository.getRemoteStoreShallowCopyShardMetadata( - snapshotId, - indexId, - new ShardId(remoteStoreIndexName, indexSetting.getUUID(), 0) - ); - assertEquals(shardShallowCopySnapshot.getRemoteStoreRepository(), remoteStoreRepositoryName); - assertEquals(shardShallowCopySnapshot.getIndexUUID(), indexSetting.getUUID()); - assertEquals(shardShallowCopySnapshot.getRepositoryBasePath(), ""); - } - - private IndexSettings getIndexSettings(String indexName) { - final IndicesService indicesService = getInstanceFromNode(IndicesService.class); - final IndexService indexService = indicesService.indexService(resolveIndex(indexName)); - return indexService.getIndexSettings(); - } - // Validate Scenario remoteStoreShallowCopy Snapshot -> remoteStoreShallowCopy Snapshot // -> remoteStoreShallowCopy Snapshot -> normal snapshot - public void testRetrieveShallowCopySnapshotCase2() throws IOException { - FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - final Client client = client(); - final String snapshotRepositoryName = "test-repo"; - final String remoteStoreRepositoryName = "test-rs-repo"; - - logger.info("--> creating snapshot repository"); - Settings snapshotRepoSettings = Settings.builder() - .put(node().settings()) - .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) - .build(); - createRepository(client, snapshotRepositoryName, snapshotRepoSettings); - - GetRepositoriesResponse updatedGetRepositoriesResponse = client.admin() - .cluster() - .prepareGetRepositories(snapshotRepositoryName) - .get(); - - RepositoryMetadata updatedRepositoryMetadata = updatedGetRepositoriesResponse.repositories().get(0); - - assertFalse(updatedRepositoryMetadata.settings().getAsBoolean(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), false)); - - logger.info("--> creating remote store repository"); - createRepository(client, remoteStoreRepositoryName); - - logger.info("--> creating an index and indexing documents"); - final String indexName = "test-idx"; - createIndex(indexName); - ensureGreen(); - indexDocuments(client, indexName); - - logger.info("--> creating a remote store enabled index and indexing documents"); - final String remoteStoreIndexName = "test-rs-idx"; - Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); - createIndex(remoteStoreIndexName, indexSettings); - indexDocuments(client, remoteStoreIndexName); - - logger.info("--> create first remote index shallow snapshot"); - - Settings snapshotRepoSettingsForShallowCopy = Settings.builder() - .put(snapshotRepoSettings) - .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) - .build(); - updateRepository(client, snapshotRepositoryName, snapshotRepoSettingsForShallowCopy); - - updatedGetRepositoriesResponse = client.admin().cluster().prepareGetRepositories(snapshotRepositoryName).get(); - - updatedRepositoryMetadata = updatedGetRepositoriesResponse.repositories().get(0); - - assertTrue(updatedRepositoryMetadata.settings().getAsBoolean(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), false)); - - CreateSnapshotResponse createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-1") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId1 = createSnapshotResponse.getSnapshotInfo().snapshotId(); - - String[] lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 1) : "lock files are " + Arrays.toString(lockFiles); - assert lockFiles[0].endsWith(snapshotId1.getUUID() + ".lock"); - - logger.info("--> create second remote index shallow snapshot"); - createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-2") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId2 = createSnapshotResponse.getSnapshotInfo().snapshotId(); - - lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 2) : "lock files are " + Arrays.toString(lockFiles); - List shallowCopySnapshotIDs = Arrays.asList(snapshotId1, snapshotId2); - for (SnapshotId snapshotId : shallowCopySnapshotIDs) { - assert lockFiles[0].contains(snapshotId.getUUID()) || lockFiles[1].contains(snapshotId.getUUID()); - } - logger.info("--> create third remote index shallow snapshot"); - createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-3") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId3 = createSnapshotResponse.getSnapshotInfo().snapshotId(); - - lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 3); - shallowCopySnapshotIDs = Arrays.asList(snapshotId1, snapshotId2, snapshotId3); - for (SnapshotId snapshotId : shallowCopySnapshotIDs) { - assert lockFiles[0].contains(snapshotId.getUUID()) - || lockFiles[1].contains(snapshotId.getUUID()) - || lockFiles[2].contains(snapshotId.getUUID()); - } - logger.info("--> create normal snapshot"); - createRepository(client, snapshotRepositoryName, snapshotRepoSettings); - createSnapshotResponse = client.admin() - .cluster() - .prepareCreateSnapshot(snapshotRepositoryName, "test-snap-4") - .setWaitForCompletion(true) - .setIndices(indexName, remoteStoreIndexName) - .get(); - final SnapshotId snapshotId4 = createSnapshotResponse.getSnapshotInfo().snapshotId(); - - lockFiles = getLockFilesInRemoteStore(remoteStoreIndexName, remoteStoreRepositoryName); - assert (lockFiles.length == 3) : "lock files are " + Arrays.toString(lockFiles); - shallowCopySnapshotIDs = Arrays.asList(snapshotId1, snapshotId2, snapshotId3); - for (SnapshotId snapshotId : shallowCopySnapshotIDs) { - assert lockFiles[0].contains(snapshotId.getUUID()) - || lockFiles[1].contains(snapshotId.getUUID()) - || lockFiles[2].contains(snapshotId.getUUID()); - } - - logger.info("--> make sure the node's repository can resolve the snapshots"); - final List originalSnapshots = Arrays.asList(snapshotId1, snapshotId2, snapshotId3, snapshotId4); - - final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); - final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(snapshotRepositoryName); - List snapshotIds = OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository) - .getSnapshotIds() - .stream() - .sorted((s1, s2) -> s1.getName().compareTo(s2.getName())) - .collect(Collectors.toList()); - assertThat(snapshotIds, equalTo(originalSnapshots)); - } - public void testReadAndWriteSnapshotsThroughIndexFile() throws Exception { final BlobStoreRepository repository = setupRepo(); final long pendingGeneration = repository.metadata.pendingGeneration(); diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java index e3b1c7af6891c..6e06e0893f37f 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -523,7 +523,7 @@ protected void indexRandomDocs(String index, int numdocs) throws InterruptedExce assertDocCount(index, numdocs); } - protected Settings getRemoteStoreBackedIndexSettings(String remoteStoreRepo) { + protected Settings getRemoteStoreBackedIndexSettings() { return Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "1") .put("index.refresh_interval", "300s") @@ -531,9 +531,6 @@ protected Settings getRemoteStoreBackedIndexSettings(String remoteStoreRepo) { .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey()) .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStoreRepo) .build(); } From 7da9ccb0d92492020e37eddf376b8909093adeaf Mon Sep 17 00:00:00 2001 From: bansvaru Date: Thu, 27 Jul 2023 18:06:32 +0530 Subject: [PATCH 5/9] update changelog Signed-off-by: bansvaru --- CHANGELOG.md | 1 + .../opensearch/remotestore/RemoteStoreBaseIntegTestCase.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12dae4fca545e..aa30fa7d1a712 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remote Segment Store Repository setting moved from `index.remote_store.repository` to `index.remote_store.segment.repository` and `cluster.remote_store.repository` to `cluster.remote_store.segment.repository` respectively for Index and Cluster level settings ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719)) - [Remote Store] Add support to restore only unassigned shards of an index ([#8792](https://github.com/opensearch-project/OpenSearch/pull/8792)) - Replace the deprecated IndexReader APIs with new storedFields() & termVectors() ([#7792](https://github.com/opensearch-project/OpenSearch/pull/7792)) +- [Remote Store] Restrict user override for remote store index level settings ([#8812](https://github.com/opensearch-project/OpenSearch/pull/8812)) ### Deprecated diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 50c3fe7261edf..4a85ff46d9025 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; @@ -99,6 +100,7 @@ public static Settings remoteStoreClusterSettings( public static Settings remoteStoreClusterSettings(String segmentRepoName, String translogRepoName) { return Settings.builder() + .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), segmentRepoName) .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), translogRepoName) From 273ad23ef0f222993fe1a92eed552d3fcf416e57 Mon Sep 17 00:00:00 2001 From: bansvaru Date: Mon, 31 Jul 2023 10:53:01 +0530 Subject: [PATCH 6/9] remove extra files creation supression in mock directories in BlobStoreRepositoryTests Signed-off-by: bansvaru --- .../blobstore/BlobStoreRepositoryRemoteIndexTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index e8b1f559ac1b8..8bd21e73a1eba 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -64,7 +64,6 @@ /** * Tests for the {@link BlobStoreRepository} and its subclasses. */ -@LuceneTestCase.SuppressFileSystems("ExtrasFS") public class BlobStoreRepositoryRemoteIndexTests extends BlobStoreRepositoryHelperTests { @Override From 8d660783438a6c06f15108d501a1b993c1fd0108 Mon Sep 17 00:00:00 2001 From: bansvaru Date: Mon, 31 Jul 2023 14:58:21 +0530 Subject: [PATCH 7/9] remove extra files creation supression in mock directories in BlobStoreRepositoryTests Signed-off-by: bansvaru --- .../BlobStoreRepositoryRemoteIndexTests.java | 1 - .../blobstore/BlobStoreRepositoryTests.java | 13 ++++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index 8bd21e73a1eba..f25498b8c8368 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -32,7 +32,6 @@ package org.opensearch.repositories.blobstore; -import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.client.Client; diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index bfd4251ecf4c1..8dad058e9b28a 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -38,15 +38,17 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; -import org.opensearch.repositories.IndexId; -import org.opensearch.repositories.RepositoriesService; -import org.opensearch.repositories.RepositoryData; -import org.opensearch.repositories.RepositoryException; -import org.opensearch.repositories.ShardGenerations; +import org.opensearch.indices.recovery.RecoverySettings; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.RepositoryPlugin; +import org.opensearch.repositories.*; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotState; @@ -54,6 +56,7 @@ import java.nio.file.Path; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; From 794d0f7d492a16053cfcbfacde0d5ed52d182abd Mon Sep 17 00:00:00 2001 From: bansvaru Date: Mon, 31 Jul 2023 19:06:29 +0530 Subject: [PATCH 8/9] use PrivateIndex setting property to avoid user overrides on remote index settings Signed-off-by: bansvaru --- .../remotestore/CreateRemoteIndexIT.java | 47 +---- .../cluster/metadata/IndexMetadata.java | 36 +++- .../metadata/MetadataCreateIndexService.java | 33 +--- .../MetadataCreateIndexServiceTests.java | 171 ++++++++---------- .../blobstore/BlobStoreRepositoryTests.java | 7 +- 5 files changed, 126 insertions(+), 168 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java index fc583942c6df4..7683651e902b2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java @@ -29,7 +29,6 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -47,7 +46,7 @@ public void teardown() { protected Settings nodeSettings(int nodeOriginal) { Settings settings = super.nodeSettings(nodeOriginal); Settings.Builder builder = Settings.builder() - .put(remoteStoreClusterSettings("my-segment-repo-1", "my-translog-repo-1", true)) + .put(remoteStoreClusterSettings("my-segment-repo-1", "my-translog-repo-1")) .put(settings); return builder.build(); } @@ -115,9 +114,8 @@ public void testRemoteStoreDisabledByUser() throws Exception { containsString( String.format( Locale.ROOT, - "Cannot override [%s] settings when [%s] is set to [true].", - SETTING_REMOTE_STORE_ENABLED, - CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() + "Validation Failed: 1: private index setting [%s] can not be set explicitly;", + SETTING_REMOTE_STORE_ENABLED ) ) ); @@ -157,9 +155,8 @@ public void testRemoteStoreEnabledByUserWithoutRemoteRepoIllegalArgumentExceptio containsString( String.format( Locale.ROOT, - "Cannot override [%s] settings when [%s] is set to [true].", - SETTING_REMOTE_STORE_ENABLED, - CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() + "Validation Failed: 1: private index setting [%s] can not be set explicitly;", + SETTING_REMOTE_STORE_ENABLED ) ) ); @@ -230,10 +227,9 @@ public void testRemoteStoreEnabledByUserWithRemoteRepoIllegalArgumentException() containsString( String.format( Locale.ROOT, - "Cannot override [%s][%s] settings when [%s] is set to [true].", + "Validation Failed: 1: private index setting [%s] can not be set explicitly;2: private index setting [%s] can not be set explicitly;", SETTING_REMOTE_STORE_ENABLED, - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, - CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() + SETTING_REMOTE_SEGMENT_STORE_REPOSITORY ) ) ); @@ -280,35 +276,10 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception { containsString( String.format( Locale.ROOT, - "Cannot override [%s][%s][%s] settings when [%s] is set to [true].", + "Validation Failed: 1: private index setting [%s] can not be set explicitly;2: private index setting [%s] can not be set explicitly;3: private index setting [%s] can not be set explicitly;", SETTING_REMOTE_STORE_ENABLED, SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, - CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() - ) - ) - ); - } - - public void testRemoteStoreOverrideReplicationTypeIndexSettings() throws Exception { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) - .build(); - IllegalArgumentException exc = expectThrows( - IllegalArgumentException.class, - () -> client().admin().indices().prepareCreate("test-idx-1").setSettings(settings).get() - ); - assertThat( - exc.getMessage(), - containsString( - String.format( - Locale.ROOT, - "To enable %s, %s should be set to %s", - SETTING_REMOTE_STORE_ENABLED, - SETTING_REPLICATION_TYPE, - ReplicationType.SEGMENT + SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY ) ) ); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 238a118e06758..f532afd120741 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -82,6 +82,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Function; @@ -285,6 +286,32 @@ public Iterator> settings() { SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT.toString(), ReplicationType::parseString, + new Setting.Validator<>() { + + @Override + public void validate(final ReplicationType value) {} + + @Override + public void validate(final ReplicationType value, final Map, Object> settings) { + final Object remoteStoreEnabled = settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING); + if (ReplicationType.SEGMENT.equals(value) == false && Objects.equals(remoteStoreEnabled, true)) { + throw new IllegalArgumentException( + "To enable " + + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey() + + ", " + + INDEX_REPLICATION_TYPE_SETTING.getKey() + + " should be set to " + + ReplicationType.SEGMENT + ); + } + } + + @Override + public Iterator> settings() { + final List> settings = List.of(INDEX_REMOTE_STORE_ENABLED_SETTING); + return settings.iterator(); + } + }, Property.IndexScope, Property.Final ); @@ -328,7 +355,8 @@ public Iterator> settings() { } }, Property.IndexScope, - Property.Final + Property.PrivateIndex, + Property.Dynamic ); /** @@ -361,7 +389,8 @@ public Iterator> settings() { } }, Property.IndexScope, - Property.Final + Property.PrivateIndex, + Property.Dynamic ); private static void validateRemoteStoreSettingEnabled(final Map, Object> settings, Setting setting) { @@ -411,7 +440,8 @@ public Iterator> settings() { } }, Property.IndexScope, - Property.Final + Property.PrivateIndex, + Property.Dynamic ); public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas"; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 4d8b70f602c50..db9964b1a2ff8 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -114,14 +114,10 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; -import java.util.stream.Stream; import static java.util.stream.Collectors.toList; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; -import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; -import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REPLICATION_TYPE_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; @@ -891,7 +887,7 @@ static Settings aggregateIndexSettings( indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); updateReplicationStrategy(indexSettingsBuilder, request.settings(), settings); - updateRemoteStoreSettings(indexSettingsBuilder, request.settings(), settings); + updateRemoteStoreSettings(indexSettingsBuilder, settings); if (sourceMetadata != null) { assert request.resizeType() != null; @@ -946,39 +942,16 @@ private static void updateReplicationStrategy(Settings.Builder settingsBuilder, /** * Updates index settings to enable remote store by default based on cluster level settings * @param settingsBuilder index settings builder to be updated with relevant settings - * @param requestSettings settings passed in during index create request * @param clusterSettings cluster level settings */ - private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, Settings requestSettings, Settings clusterSettings) { - boolean isRemoteStoreClusterEnabled = CLUSTER_REMOTE_STORE_ENABLED_SETTING.get(clusterSettings); - List overriddenSettings = getRemoteStoreOverriddenSetting(requestSettings); - if (overriddenSettings.isEmpty() == false) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "Cannot override [%s] settings when [%s] is set to [%s].", - String.join("][", overriddenSettings), - CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), - isRemoteStoreClusterEnabled - ) - ); - } - - if (isRemoteStoreClusterEnabled == true) { + private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, Settings clusterSettings) { + if (CLUSTER_REMOTE_STORE_ENABLED_SETTING.get(clusterSettings) == true) { settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(clusterSettings)) .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(clusterSettings)); } } - private static List getRemoteStoreOverriddenSetting(Settings requestSettings) { - return Stream.of( - INDEX_REMOTE_STORE_ENABLED_SETTING, - INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING, - INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING - ).filter(setting -> setting.exists(requestSettings)).map(Setting::getKey).collect(toList()); - } - public static void validateStoreTypeSettings(Settings settings) { // deprecate simplefs store type: if (IndexModule.Type.SIMPLEFS.match(IndexModule.INDEX_STORE_TYPE_SETTING.get(settings))) { diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 2b80c2a139e2c..63c3511a97d2b 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -114,7 +114,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; @@ -1256,123 +1255,103 @@ public void testRemoteStoreNoUserOverrideIndexSettings() { } public void testRemoteStoreDisabledByUserIndexSettings() { - Settings settings = Settings.builder() - .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") - .build(); - FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); final Settings.Builder requestSettings = Settings.builder(); requestSettings.put(SETTING_REMOTE_STORE_ENABLED, false); - request.settings(requestSettings.build()); - IllegalArgumentException exc = expectThrows( - IllegalArgumentException.class, - () -> aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, + withTemporaryClusterService(((clusterService, threadPool) -> { + MetadataCreateIndexService checkerService = new MetadataCreateIndexService( Settings.EMPTY, + clusterService, null, - settings, + null, + null, + createTestShardLimitService(randomIntBetween(1, 1000), false, clusterService), + new Environment(Settings.builder().put("path.home", "dummy").build(), null), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() - ) - ); - assertThat( - exc.getMessage(), - containsString( - String.format( - Locale.ROOT, - "Cannot override [%s] settings when [%s] is set to [true].", - SETTING_REMOTE_STORE_ENABLED, - CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() - ) - ) - ); + threadPool, + null, + new SystemIndices(Collections.emptyMap()), + true, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()) + ); + + final List validationErrors = checkerService.getIndexSettingsValidationErrors( + requestSettings.build(), + true, + Optional.empty() + ); + assertThat(validationErrors.size(), is(1)); + assertThat( + validationErrors.get(0), + is(String.format(Locale.ROOT, "expected [%s] to be private but it was not", SETTING_REMOTE_STORE_ENABLED)) + ); + })); } public void testRemoteStoreOverrideSegmentRepoIndexSettings() { - Settings settings = Settings.builder() - .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") - .build(); - FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); final Settings.Builder requestSettings = Settings.builder(); requestSettings.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(SETTING_REMOTE_STORE_ENABLED, true) .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo"); - request.settings(requestSettings.build()); - IllegalArgumentException exc = expectThrows( - IllegalArgumentException.class, - () -> aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, + withTemporaryClusterService(((clusterService, threadPool) -> { + MetadataCreateIndexService checkerService = new MetadataCreateIndexService( Settings.EMPTY, + clusterService, + null, + null, null, - settings, + createTestShardLimitService(randomIntBetween(1, 1000), false, clusterService), + new Environment(Settings.builder().put("path.home", "dummy").build(), null), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() - ) - ); - assertThat( - exc.getMessage(), - containsString( - String.format( - Locale.ROOT, - "Cannot override [%s][%s] settings when [%s] is set to [true].", - SETTING_REMOTE_STORE_ENABLED, - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, - CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() - ) - ) - ); + threadPool, + null, + new SystemIndices(Collections.emptyMap()), + true, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()) + ); + + final List validationErrors = checkerService.getIndexSettingsValidationErrors( + requestSettings.build(), + true, + Optional.empty() + ); + assertThat(validationErrors.size(), is(1)); + assertThat( + validationErrors.get(0), + is(String.format(Locale.ROOT, "expected [%s] to be private but it was not", SETTING_REMOTE_SEGMENT_STORE_REPOSITORY)) + ); + })); } public void testRemoteStoreOverrideTranslogRepoIndexSettings() { - Settings settings = Settings.builder() - .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") - .build(); - FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); final Settings.Builder requestSettings = Settings.builder(); requestSettings.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo"); - request.settings(requestSettings.build()); - IllegalArgumentException exc = expectThrows( - IllegalArgumentException.class, - () -> aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, + withTemporaryClusterService(((clusterService, threadPool) -> { + MetadataCreateIndexService checkerService = new MetadataCreateIndexService( Settings.EMPTY, + clusterService, + null, null, - settings, + null, + createTestShardLimitService(randomIntBetween(1, 1000), false, clusterService), + new Environment(Settings.builder().put("path.home", "dummy").build(), null), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() - ) - ); - assertThat( - exc.getMessage(), - containsString( - String.format( - Locale.ROOT, - "Cannot override [%s] settings when [%s] is set to [true].", - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, - CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey() - ) - ) - ); + threadPool, + null, + new SystemIndices(Collections.emptyMap()), + true, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()) + ); + + final List validationErrors = checkerService.getIndexSettingsValidationErrors( + requestSettings.build(), + true, + Optional.empty() + ); + assertThat(validationErrors.size(), is(1)); + assertThat( + validationErrors.get(0), + is(String.format(Locale.ROOT, "expected [%s] to be private but it was not", SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY)) + ); + })); } public void testBuildIndexMetadata() { diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index 8dad058e9b28a..2dbf8812cdd7e 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -48,7 +48,12 @@ import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.RepositoryPlugin; -import org.opensearch.repositories.*; +import org.opensearch.repositories.IndexId; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.Repository; +import org.opensearch.repositories.RepositoryData; +import org.opensearch.repositories.RepositoryException; +import org.opensearch.repositories.ShardGenerations; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotState; From 839aa944234ad7ee1e64ee99b9f3fa8937081239 Mon Sep 17 00:00:00 2001 From: bansvaru Date: Mon, 31 Jul 2023 22:51:23 +0530 Subject: [PATCH 9/9] fix UTs Signed-off-by: bansvaru --- .../opensearch/snapshots/RestoreService.java | 7 +-- .../TransportRemoteStoreStatsActionTests.java | 3 + .../opensearch/index/IndexSettingsTests.java | 37 +------------ ...oteRefreshSegmentPressureServiceTests.java | 6 +- .../index/seqno/ReplicationTrackerTests.java | 55 +++++++++++++++---- .../index/shard/IndexShardTests.java | 4 +- .../index/translog/RemoteFSTranslogTests.java | 2 + 7 files changed, 60 insertions(+), 54 deletions(-) diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 9b32a97aa5c6c..d7e89172c5837 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -227,12 +227,7 @@ public ClusterState execute(ClusterState currentState) { List indicesToBeRestored = new ArrayList<>(); int totalShards = 0; - List filteredIndices = filterIndices( - Arrays.asList(request.indices()), - currentState.metadata().indices().keySet().toArray(new String[0]), - IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN - ); - for (String index : filteredIndices) { + for (String index : request.indices()) { IndexMetadata currentIndexMetadata = currentState.metadata().index(index); if (currentIndexMetadata == null) { // ToDo: Handle index metadata does not exist case. (GitHub #3457) diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java index 25e44884814a5..5c17765903504 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java @@ -32,6 +32,7 @@ import org.opensearch.index.remote.RemoteRefreshSegmentTracker; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.indices.IndicesService; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.transport.MockTransport; import org.opensearch.transport.TransportService; @@ -45,6 +46,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; @@ -70,6 +72,7 @@ public void setUp() throws Exception { remoteStoreIndexMetadata = IndexMetadata.builder(INDEX.getName()) .settings( settings(Version.CURRENT).put(SETTING_INDEX_UUID, INDEX.getUUID()) + .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(SETTING_REMOTE_STORE_ENABLED, true) .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-test-repo") .build() diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index 6207d2f2725ad..97ecc82f04c14 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -779,6 +779,7 @@ public void testRemoteStoreExplicitSetting() { "index", Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) .build() ); @@ -795,22 +796,6 @@ public void testRemoteTranslogStoreDefaultSetting() { assertFalse(settings.isRemoteTranslogStoreEnabled()); } - public void testUpdateRemoteStoreFails() { - Set> remoteStoreSettingSet = new HashSet<>(); - remoteStoreSettingSet.add(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING); - IndexScopedSettings settings = new IndexScopedSettings(Settings.EMPTY, remoteStoreSettingSet); - SettingsException error = expectThrows( - SettingsException.class, - () -> settings.updateSettings( - Settings.builder().put("index.remote_store.enabled", randomBoolean()).build(), - Settings.builder(), - Settings.builder(), - "index" - ) - ); - assertEquals(error.getMessage(), "final index setting [index.remote_store.enabled], not updateable"); - } - public void testEnablingRemoteStoreFailsWhenReplicationTypeIsDocument() { Settings indexSettings = Settings.builder() .put("index.replication.type", ReplicationType.DOCUMENT) @@ -846,6 +831,7 @@ public void testRemoteRepositoryExplicitSetting() { "index", Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "repo1") .build() @@ -854,25 +840,6 @@ public void testRemoteRepositoryExplicitSetting() { assertEquals("repo1", settings.getRemoteStoreRepository()); } - public void testUpdateRemoteRepositoryFails() { - Set> remoteStoreSettingSet = new HashSet<>(); - remoteStoreSettingSet.add(IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING); - IndexScopedSettings settings = new IndexScopedSettings(Settings.EMPTY, remoteStoreSettingSet); - SettingsException error = expectThrows( - SettingsException.class, - () -> settings.updateSettings( - Settings.builder().put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, randomUnicodeOfLength(10)).build(), - Settings.builder(), - Settings.builder(), - "index" - ) - ); - assertEquals( - error.getMessage(), - String.format(Locale.ROOT, "final index setting [%s], not updateable", IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY) - ); - } - public void testSetRemoteRepositoryFailsWhenRemoteStoreIsNotEnabled() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java index 5ccacd4048596..69f3456e825f8 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java @@ -16,6 +16,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; @@ -150,7 +151,10 @@ public void testValidateSegmentUploadLag() { } private static IndexShard createIndexShard(ShardId shardId, boolean remoteStoreEnabled) { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, String.valueOf(remoteStoreEnabled)).build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, String.valueOf(remoteStoreEnabled)) + .build(); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test_index", settings); IndexShard indexShard = mock(IndexShard.class); when(indexShard.indexSettings()).thenReturn(indexSettings); diff --git a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java index 03a6fc3df824d..5e88b892cc835 100644 --- a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java @@ -1293,7 +1293,10 @@ public void testGlobalCheckpointUpdateWithRemoteTranslogEnabled() { assertThat(allocations.size(), equalTo(active.size() + initializing.size())); final AllocationId primaryId = active.iterator().next(); - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") + .build(); final ReplicationTracker tracker = newTracker(primaryId, settings); assertThat(tracker.getGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO)); @@ -1368,7 +1371,10 @@ public void testUpdateFromClusterManagerWithRemoteTranslogEnabled() { assertThat(allocations.size(), equalTo(active.size() + initializing.size())); final AllocationId primaryId = active.iterator().next(); - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") + .build(); final ReplicationTracker tracker = newTracker(primaryId, settings); assertThat(tracker.getGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO)); @@ -1438,7 +1444,10 @@ public void testUpdateFromClusterManagerWithRemoteTranslogEnabled() { */ public void testUpdateGlobalCheckpointOnReplicaWithRemoteTranslogEnabled() { final AllocationId active = AllocationId.newInitializing(); - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") + .build(); final ReplicationTracker tracker = newTracker(active, settings); final long globalCheckpoint = randomLongBetween(NO_OPS_PERFORMED, Long.MAX_VALUE - 1); tracker.updateGlobalCheckpointOnReplica(globalCheckpoint, "test"); @@ -1460,7 +1469,10 @@ public void testMarkAllocationIdAsInSyncWithRemoteTranslogEnabled() throws Excep Set initializing = new HashSet<>(initializingWithCheckpoints.keySet()); final AllocationId primaryId = active.iterator().next(); final AllocationId replicaId = initializing.iterator().next(); - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") + .build(); final ReplicationTracker tracker = newTracker(primaryId, settings); tracker.updateFromClusterManager(initialClusterStateVersion, ids(active), routingTable(initializing, primaryId)); final long localCheckpoint = randomLongBetween(0, Long.MAX_VALUE - 1); @@ -1485,7 +1497,10 @@ public void testMissingActiveIdsDoesNotPreventAdvanceWithRemoteTranslogEnabled() assigned.putAll(active); assigned.putAll(initializing); AllocationId primaryId = active.keySet().iterator().next(); - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") + .build(); final ReplicationTracker tracker = newTracker(primaryId, settings); tracker.updateFromClusterManager(randomNonNegativeLong(), ids(active.keySet()), routingTable(initializing.keySet(), primaryId)); tracker.activatePrimaryMode(NO_OPS_PERFORMED); @@ -1515,7 +1530,10 @@ public void testMissingInSyncIdsDoesNotPreventAdvanceWithRemoteTranslogEnabled() logger.info("active: {}, initializing: {}", active, initializing); AllocationId primaryId = active.keySet().iterator().next(); - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") + .build(); final ReplicationTracker tracker = newTracker(primaryId, settings); tracker.updateFromClusterManager(randomNonNegativeLong(), ids(active.keySet()), routingTable(initializing.keySet(), primaryId)); tracker.activatePrimaryMode(NO_OPS_PERFORMED); @@ -1540,7 +1558,10 @@ public void testInSyncIdsAreIgnoredIfNotValidatedByClusterManagerWithRemoteTrans final Map initializing = randomAllocationsWithLocalCheckpoints(1, 5); final Map nonApproved = randomAllocationsWithLocalCheckpoints(1, 5); final AllocationId primaryId = active.keySet().iterator().next(); - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") + .build(); final ReplicationTracker tracker = newTracker(primaryId, settings); tracker.updateFromClusterManager(randomNonNegativeLong(), ids(active.keySet()), routingTable(initializing.keySet(), primaryId)); tracker.activatePrimaryMode(NO_OPS_PERFORMED); @@ -1578,7 +1599,10 @@ public void testInSyncIdsAreRemovedIfNotValidatedByClusterManagerWithRemoteTrans if (randomBoolean()) { allocations.putAll(initializingToBeRemoved); } - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") + .build(); final ReplicationTracker tracker = newTracker(primaryId, settings); tracker.updateFromClusterManager(initialClusterStateVersion, ids(active), routingTable(initializing, primaryId)); tracker.activatePrimaryMode(NO_OPS_PERFORMED); @@ -1624,7 +1648,10 @@ public void testUpdateAllocationIdsFromClusterManagerWithRemoteTranslogEnabled() final Set initializingIds = activeAndInitializingAllocationIds.v2(); AllocationId primaryId = activeAllocationIds.iterator().next(); IndexShardRoutingTable routingTable = routingTable(initializingIds, primaryId); - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") + .build(); final ReplicationTracker tracker = newTracker(primaryId, settings); tracker.updateFromClusterManager(initialClusterStateVersion, ids(activeAllocationIds), routingTable); tracker.activatePrimaryMode(NO_OPS_PERFORMED); @@ -1936,7 +1963,10 @@ public void testSegmentReplicationCheckpointTrackingInvalidAllocationIDs() { } public void testPrimaryContextHandoffWithRemoteTranslogEnabled() throws IOException { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") + .build(); final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings); final ShardId shardId = new ShardId("test", "_na_", 0); @@ -2115,7 +2145,10 @@ public void testPrimaryContextHandoffWithRemoteTranslogEnabled() throws IOExcept public void testIllegalStateExceptionIfUnknownAllocationIdWithRemoteTranslogEnabled() { final AllocationId active = AllocationId.newInitializing(); final AllocationId initializing = AllocationId.newInitializing(); - Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true").build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") + .build(); final ReplicationTracker tracker = newTracker(active, settings); tracker.updateFromClusterManager( randomNonNegativeLong(), diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index 915a15da6cb1d..e04b5a5e77ac9 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -1260,6 +1260,7 @@ public void testGetChangesSnapshotThrowsAssertForRemoteStore() throws IOExceptio .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) .build(); final IndexMetadata.Builder indexMetadata = IndexMetadata.builder(shardRouting.getIndexName()).settings(settings).primaryTerm(0, 1); @@ -4839,12 +4840,13 @@ public void testTranslogFactoryForRemoteTranslogBackedReplicaShard() throws IOEx .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "seg-test") .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "txlog-test") .build(); final IndexShard replicaShard = newStartedShard(false, primarySettings, new NRTReplicationEngineFactory()); - assertEquals(replicaShard.getEngine().getClass(), InternalEngine.class); + assertEquals(replicaShard.getEngine().getClass(), NRTReplicationEngine.class); assertEquals(replicaShard.getEngine().config().getTranslogFactory().getClass(), InternalTranslogFactory.class); closeShards(replicaShard); } diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java index d26379eaefa5c..c8933f1fabaae 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java @@ -47,6 +47,7 @@ import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.indices.recovery.RecoverySettings; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.BlobStoreTestUtil; import org.opensearch.repositories.fs.FsRepository; @@ -182,6 +183,7 @@ private TranslogConfig getTranslogConfig(final Path path) { // only randomize between nog age retention and a long one, so failures will have a chance of reproducing .put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomBoolean() ? "-1ms" : "1h") .put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), randomIntBetween(-1, 2048) + "b") + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) .build(); return getTranslogConfig(path, settings);