update chunking index setting and chunking processor setting

Signed-off-by: yuye-aws <yuyezhu@amazon.com>
opensearch-project · Apr 28, 2024 · 5c45e66 · 5c45e66
1 parent f3a81f6
commit 5c45e66
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 4 deletions.
diff --git a/qa/restart-upgrade/src/test/resources/processor/ChunkingIndexSettings.json b/qa/restart-upgrade/src/test/resources/processor/ChunkingIndexSettings.json
@@ -1,8 +1,8 @@
 {
   "settings":{
     "default_pipeline": "%s",
-    "number_of_shards": 3,
-    "number_of_replicas": 1
+    "number_of_shards": 1,
+    "number_of_replicas": 0
   },
   "mappings": {
     "properties": {

diff --git a/qa/rolling-upgrade/src/test/resources/processor/ChunkingIndexSettings.json b/qa/rolling-upgrade/src/test/resources/processor/ChunkingIndexSettings.json
@@ -1,8 +1,8 @@
 {
   "settings":{
     "default_pipeline": "%s",
-    "number_of_shards": 3,
-    "number_of_replicas": 1
+    "number_of_shards": 1,
+    "number_of_replicas": 0
   },
   "mappings": {
     "properties": {

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextChunkingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextChunkingProcessor.java
@@ -162,6 +162,7 @@ private int getMaxTokenCount(final Map<String, Object> sourceAndMetadataMap) {
      */
     @Override
     public IngestDocument execute(final IngestDocument ingestDocument) {
+        /*
         Map<String, Object> sourceAndMetadataMap = ingestDocument.getSourceAndMetadata();
         validateFieldsValue(sourceAndMetadataMap);
         // fixed token length algorithm needs runtime parameter max_token_count for tokenization
@@ -170,6 +171,7 @@ public IngestDocument execute(final IngestDocument ingestDocument) {
         runtimeParameters.put(FixedTokenLengthChunker.MAX_TOKEN_COUNT_FIELD, maxTokenCount);
         runtimeParameters.put(MAX_CHUNK_LIMIT_FIELD, maxChunkLimit);
         chunkMapType(sourceAndMetadataMap, fieldMap, runtimeParameters, 0);
+        */
         return ingestDocument;
     }