Skip to content

Commit

Permalink
introducing new cluster block for remote snapshot
Browse files Browse the repository at this point in the history 
Signed-off-by: Vishal Sarda <vsarda@amazon.com>
  • Loading branch information
@Vishalks
Vishalks committed Oct 26, 2022
1 parent f5f1f16 commit e2f7b25
Show file tree
Hide file tree
Showing 4 changed files with 68 additions and 66 deletions.
119 changes: 54 additions & 65 deletions server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,7 @@
import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
import org.opensearch.action.delete.DeleteRequest;
import org.opensearch.action.get.GetRequest;
import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
import org.opensearch.action.index.IndexRequestBuilder;
import org.opensearch.client.Client;
import org.opensearch.cluster.ClusterState;
Expand All @@ -21,6 +19,7 @@
import org.opensearch.cluster.routing.GroupShardsIterator;
import org.opensearch.cluster.routing.ShardIterator;
import org.opensearch.cluster.routing.ShardRouting;
import org.opensearch.common.collect.Map;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.util.FeatureFlags;
Expand All @@ -31,11 +30,11 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.ExecutionException;

import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.contains;
import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.FS;
import static org.opensearch.common.util.CollectionUtils.iterableAsArrayList;

Expand Down Expand Up @@ -110,83 +109,73 @@ public void testCreateSearchableSnapshot() throws Exception {
assertIndexDirectoryDoesNotExist("test-idx-1-copy", "test-idx-2-copy");
}

public void testIndexSearchableSnapshot() throws Exception {
disableRepoConsistencyCheck("reason");
boolean exceptionThrown = false;
final int numReplicasIndex1 = randomIntBetween(1, 4);
internalCluster().ensureAtLeastNumDataNodes(Math.max(numReplicasIndex1, 0) + 1);
public void testSearchableSnapshotIndexIsReadOnly() throws Exception {
final String indexName = "test-index";
final Client client = client();
String index = "test-idx-4";
createRepository("test-repo", "fs");
createIndex(
index,
indexName,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, Integer.toString(numReplicasIndex1))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "0")
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1")
.put(IndexMetadata.APIBlock.WRITE.settingName(), true)
.build()
);
ensureGreen();
IndexRequestBuilder builder = client().prepareIndex(index).setId(Integer.toString(1)).setSource("field1", "bar ");
try {
client().index(builder.request()).get(); // should throw exception
assertTrue(false); // this should never be called
} catch (ExecutionException e) {
assertEquals(ClusterBlockException.class, e.getCause().getClass());
}

}
indexRandomDocs("test-idx-1", 100);

public void testDeleteDocSearchableSnapshot() throws Exception {
disableRepoConsistencyCheck("reason");
boolean exceptionThrown = false;
final int numReplicasIndex1 = randomIntBetween(1, 4);
internalCluster().ensureAtLeastNumDataNodes(Math.max(numReplicasIndex1, 0) + 1);
String index = "test-idx-5";
createRepository("test-repo", "fs");
createIndex(
index,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, Integer.toString(numReplicasIndex1))
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1")
.put(IndexMetadata.APIBlock.WRITE.settingName(), true)
.build()
logger.info("--> snapshot");
final CreateSnapshotResponse createSnapshotResponse = client.admin()
.cluster()
.prepareCreateSnapshot("test-repo", "test-snap")
.setWaitForCompletion(true)
.setIndices(indexName)
.get();
MatcherAssert.assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
MatcherAssert.assertThat(
createSnapshotResponse.getSnapshotInfo().successfulShards(),
equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())
);

assertTrue(client.admin().indices().prepareDelete(indexName).get().isAcknowledged());

logger.info("--> restore indices as 'remote_snapshot'");
client.admin()
.cluster()
.prepareRestoreSnapshot("test-repo", "test-snap")
.setRenamePattern("(.+)")
.setRenameReplacement("$1")
.setStorageType(RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT)
.setWaitForCompletion(true)
.execute()
.actionGet();
ensureGreen();
String docId = "1";
client().prepareIndex(index).setId(docId).setSource("field1", "bar ");

assertIndexingBlocked(indexName);
assertIndexSettingChangeBlocked(indexName);
assertTrue(client.admin().indices().prepareDelete(indexName).get().isAcknowledged());
assertThrows("Expect index to not exist", IndexNotFoundException.class,
() -> client.admin().indices().prepareGetIndex().setIndices(indexName).execute().actionGet());
}

private void assertIndexingBlocked(String index) {
try {
client().delete(new DeleteRequest(index, docId)).get(); // should throw exception
assertTrue(false); // this should never be called
} catch (ExecutionException e) {
assertEquals(ClusterBlockException.class, e.getCause().getClass());
final IndexRequestBuilder builder = client().prepareIndex(index);
builder.setSource("foo", "bar");
builder.execute().actionGet();
fail("Expected operation to throw an exception");
} catch (ClusterBlockException e) {
MatcherAssert.assertThat(e.blocks(), contains(IndexMetadata.REMOTE_READ_ONLY_ALLOW_DELETE));
}
}

public void testDeleteIndexSearchableSnapshot() throws Exception {
disableRepoConsistencyCheck("reason");
boolean exceptionThrown = false;
final int numReplicasIndex1 = randomIntBetween(1, 4);
internalCluster().ensureAtLeastNumDataNodes(Math.max(numReplicasIndex1, 0) + 1);
String index = "test-idx-6";
createRepository("test-repo", "fs");
createIndex(
index,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, Integer.toString(numReplicasIndex1))
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1")
.put(IndexMetadata.APIBlock.WRITE.settingName(), true)
.build()
);
ensureGreen();
String docId = "1";
client().prepareIndex(index).setId(docId).setSource("field1", "bar ");
client().admin().indices().delete(new DeleteIndexRequest(index)).get();
private void assertIndexSettingChangeBlocked(String index) {
try {
client().get(new GetRequest(index, docId)).get(); // throws exception because the index is deleted.
assertTrue(false); // this should never be called
} catch (ExecutionException e) {
assertEquals(IndexNotFoundException.class, e.getCause().getClass());
final UpdateSettingsRequestBuilder builder = client().admin().indices().prepareUpdateSettings(index);
builder.setSettings(Map.of("index.refresh_interval", 10));
builder.execute().actionGet();
fail("Expected operation to throw an exception");
} catch (ClusterBlockException e) {
MatcherAssert.assertThat(e.blocks(), contains(IndexMetadata.REMOTE_READ_ONLY_ALLOW_DELETE));
}
}

Expand Down
4 changes: 4 additions & 0 deletions server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.common.util.set.Sets;
import org.opensearch.index.IndexModule;
import org.opensearch.rest.RestStatus;

import java.io.IOException;
Expand Down Expand Up @@ -393,6 +394,9 @@ public Builder addBlocks(IndexMetadata indexMetadata) {
if (IndexMetadata.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.get(indexMetadata.getSettings())) {
addIndexBlock(indexName, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK);
}
if (IndexModule.Type.REMOTE_SNAPSHOT.getSettingsKey().equals(indexMetadata.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()))) {
addIndexBlock(indexName, IndexMetadata.REMOTE_READ_ONLY_ALLOW_DELETE);
}
return this;
}

Expand Down
10 changes: 10 additions & 0 deletions server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,16 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
EnumSet.of(ClusterBlockLevel.METADATA_WRITE, ClusterBlockLevel.WRITE)
);

public static final ClusterBlock REMOTE_READ_ONLY_ALLOW_DELETE = new ClusterBlock(
13,
"remote index is read-only",
false,
false,
true,
RestStatus.FORBIDDEN,
EnumSet.of(ClusterBlockLevel.METADATA_WRITE, ClusterBlockLevel.WRITE)
);

/**
* The state of the index.
*
Expand Down
1 change: 0 additions & 1 deletion server/src/main/java/org/opensearch/snapshots/RestoreService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1227,7 +1227,6 @@ private static IndexMetadata addSnapshotToIndexSettings(IndexMetadata metadata,
.put(IndexSettings.SEARCHABLE_SNAPSHOT_ID_UUID.getKey(), snapshot.getSnapshotId().getUUID())
.put(IndexSettings.SEARCHABLE_SNAPSHOT_ID_NAME.getKey(), snapshot.getSnapshotId().getName())
.put(IndexSettings.SEARCHABLE_SNAPSHOT_INDEX_ID.getKey(), indexId.getId())
.put(IndexMetadata.APIBlock.WRITE.settingName(), true)
.put(metadata.getSettings())
.build();
return IndexMetadata.builder(metadata).settings(newSettings).build();
Expand Down

0 comments on commit e2f7b25

Please sign in to comment.