Introduce remote store path type in customData in IndexMetadata

Signed-off-by: Ashish Singh <ssashish@amazon.com>
opensearch-project · Mar 12, 2024 · 5e915e1 · 5e915e1
1 parent e6eec36
commit 5e915e1
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 2 deletions.
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
@@ -635,6 +635,7 @@ public static APIBlock readFrom(StreamInput input) throws IOException {
     static final String KEY_ROLLOVER_INFOS = "rollover_info";
     static final String KEY_SYSTEM = "system";
     public static final String KEY_PRIMARY_TERMS = "primary_terms";
+    public static final String REMOTE_STORE_CUSTOM_KEY = "remote_store";
 
     public static final String INDEX_STATE_FILE_PREFIX = "state-";
 

diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -88,6 +88,7 @@
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.MapperService.MergeReason;
 import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.index.remote.RemoteStorePathType;
 import org.opensearch.index.shard.IndexSettingProvider;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.IndexCreationException;
@@ -498,7 +499,8 @@ private ClusterState applyCreateIndexWithTemporaryService(
                     temporaryIndexMeta.getSettings(),
                     temporaryIndexMeta.getRoutingNumShards(),
                     sourceMetadata,
-                    temporaryIndexMeta.isSystem()
+                    temporaryIndexMeta.isSystem(),
+                    temporaryIndexMeta.getCustomData()
                 );
             } catch (Exception e) {
                 logger.info("failed to build index metadata [{}]", request.index());
@@ -544,6 +546,16 @@ private IndexMetadata buildAndValidateTemporaryIndexMetadata(
         tmpImdBuilder.settings(indexSettings);
         tmpImdBuilder.system(isSystem);
 
+        if (isRemoteStoreAttributePresent(settings)) {
+            String pathType;
+            if (clusterService.getClusterSettings().get(IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_OPTIMISED_SETTING)) {
+                pathType = RemoteStorePathType.HASHED_PREFIX.toString();
+            } else {
+                pathType = RemoteStorePathType.FIXED.toString();
+            }
+            tmpImdBuilder.putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, Map.of(RemoteStorePathType.NAME, pathType));
+        }
+
         // Set up everything, now locally create the index to see that things are ok, and apply
         IndexMetadata tempMetadata = tmpImdBuilder.build();
         validateActiveShardCount(request.waitForActiveShards(), tempMetadata);
@@ -1147,7 +1159,8 @@ static IndexMetadata buildIndexMetadata(
         Settings indexSettings,
         int routingNumShards,
         @Nullable IndexMetadata sourceMetadata,
-        boolean isSystem
+        boolean isSystem,
+        Map<String, DiffableStringMap> customData
     ) {
         IndexMetadata.Builder indexMetadataBuilder = createIndexMetadataBuilder(indexName, sourceMetadata, indexSettings, routingNumShards);
         indexMetadataBuilder.system(isSystem);
@@ -1168,6 +1181,10 @@ static IndexMetadata buildIndexMetadata(
             indexMetadataBuilder.putAlias(aliases.get(i));
         }
 
+        for (Map.Entry<String, DiffableStringMap> entry : customData.entrySet()) {
+            indexMetadataBuilder.putCustom(entry.getKey(), entry.getValue());
+        }
+
         indexMetadataBuilder.state(IndexMetadata.State.OPEN);
         return indexMetadataBuilder.build();
     }

diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -709,6 +709,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 CpuBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT,
                 CpuBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT,
                 IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING,
+                IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_OPTIMISED_SETTING,
 
                 // Concurrent segment search settings
                 SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING,

diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathType.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathType.java
@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+/**
+ * Enumerates the types of remote store paths resolution techniques supported by OpenSearch.
+ * For more information, see <a href="https://github.com/opensearch-project/OpenSearch/issues/12567">Github issue #12567</a>.
+ *
+ * @opensearch.internal
+ */
+public enum RemoteStorePathType {
+
+    FIXED,
+    HASHED_PREFIX;
+
+    public static RemoteStorePathType parseString(String remoteStorePathType) {
+        try {
+            return RemoteStorePathType.valueOf(remoteStorePathType);
+        } catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Could not parse RemoteStorePathType for [" + remoteStorePathType + "]");
+        } catch (NullPointerException npe) {
+            // return a default value for null input
+            return FIXED;
+        }
+    }
+
+    /**
+     * This string is used as key for storing information in the custom data in index settings.
+     */
+    public static final String NAME = "path_type";
+}
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -314,6 +314,17 @@ public class IndicesService extends AbstractLifecycleComponent
         Property.Final
     );
 
+    /**
+     * This setting is used to enable the optimisation in prefix path which helps in achieving higher throughput and lesser
+     * rate limiting by remote store providers. This setting is effective only for remote store enabled cluster.
+     */
+    public static final Setting<Boolean> CLUSTER_REMOTE_STORE_PATH_PREFIX_OPTIMISED_SETTING = Setting.boolSetting(
+        "cluster.remote_store.index.path.prefix.optimised",
+        true,
+        Property.NodeScope,
+        Property.Dynamic
+    );
+
     /**
      * The node's settings.
      */