Ignore all malformed objects when ignore_malformed is true (opensearc…

…h-project#4494) Fixes a bug to not fail the entire document when "ignore_malformed" is set to true. Allowing the valid fields to be indexed and ignore only the malformed fields. Signed-off-by: Hauck <joaoh14@gmail.com> Signed-off-by: Braddy Yeoh <braddy.yeoh@ucdconnect.ie>
yeohbraddy · Oct 2, 2022 · 532496f · 532496f
1 parent 51a529f
commit 532496f
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 11 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -64,6 +64,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - `opensearch.bat` fails to execute when install path includes spaces ([#4362](https://github.com/opensearch-project/OpenSearch/pull/4362))
 - Getting security exception due to access denied 'java.lang.RuntimePermission' 'accessDeclaredMembers' when trying to get snapshot with S3 IRSA ([#4469](https://github.com/opensearch-project/OpenSearch/pull/4469))
 - Fixed flaky test `ResourceAwareTasksTests.testTaskIdPersistsInThreadContext` ([#4484](https://github.com/opensearch-project/OpenSearch/pull/4484))
+- Fixed the ignore_malformed setting to also ignore objects ([#4494](https://github.com/opensearch-project/OpenSearch/pull/4494))
 
 ### Security
 - CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))

diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java
@@ -34,6 +34,7 @@
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
@@ -268,6 +269,8 @@ public void parse(ParseContext context) throws IOException {
         try {
             parseCreateField(context);
         } catch (Exception e) {
+            boolean ignore_malformed = false;
+            if (context.indexSettings() != null) ignore_malformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
             String valuePreview = "";
             try {
                 XContentParser parser = context.parser();
@@ -278,23 +281,27 @@ public void parse(ParseContext context) throws IOException {
                     valuePreview = complexValue.toString();
                 }
             } catch (Exception innerException) {
+                if (ignore_malformed == false) {
+                    throw new MapperParsingException(
+                        "failed to parse field [{}] of type [{}] in document with id '{}'. " + "Could not parse field value preview,",
+                        e,
+                        fieldType().name(),
+                        fieldType().typeName(),
+                        context.sourceToParse().id()
+                    );
+                }
+            }
+
+            if (ignore_malformed == false) {
                 throw new MapperParsingException(
-                    "failed to parse field [{}] of type [{}] in document with id '{}'. " + "Could not parse field value preview,",
+                    "failed to parse field [{}] of type [{}] in document with id '{}'. " + "Preview of field's value: '{}'",
                     e,
                     fieldType().name(),
                     fieldType().typeName(),
-                    context.sourceToParse().id()
+                    context.sourceToParse().id(),
+                    valuePreview
                 );
             }
-
-            throw new MapperParsingException(
-                "failed to parse field [{}] of type [{}] in document with id '{}'. " + "Preview of field's value: '{}'",
-                e,
-                fieldType().name(),
-                fieldType().typeName(),
-                context.sourceToParse().id(),
-                valuePreview
-            );
         }
         multiFields.parse(this, context);
     }