Skip to content

Commit

Permalink
Ignore all malformed objects when ignore_malformed is true (#4494)
Browse files Browse the repository at this point in the history
Fixes a bug to not fail the entire document when "ignore_malformed" is set to true. Allowing the valid fields
to be indexed and ignore only the malformed fields.

Signed-off-by: Hauck <joaoh14@gmail.com>
(cherry picked from commit 45c9597)
  • Loading branch information
hauck-jvsh authored and github-actions[bot] committed Sep 15, 2022
1 parent 3612b24 commit 5e44c84
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 11 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- [Bug]: gradle check failing with java heap OutOfMemoryError (([#4328](https://github.com/opensearch-project/OpenSearch/
- `opensearch.bat` fails to execute when install path includes spaces ([#4362](https://github.com/opensearch-project/OpenSearch/pull/4362))
- Fixed flaky test `ResourceAwareTasksTests.testTaskIdPersistsInThreadContext` ([#4484](https://github.com/opensearch-project/OpenSearch/pull/4484))
- Fixed the ignore_malformed setting to also ignore objects ([#4494](https://github.com/opensearch-project/OpenSearch/pull/4494))

### Security
- CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))
Expand Down
29 changes: 18 additions & 11 deletions server/src/main/java/org/opensearch/index/mapper/FieldMapper.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;

import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
Expand Down Expand Up @@ -268,6 +269,8 @@ public void parse(ParseContext context) throws IOException {
try {
parseCreateField(context);
} catch (Exception e) {
boolean ignore_malformed = false;
if (context.indexSettings() != null) ignore_malformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
String valuePreview = "";
try {
XContentParser parser = context.parser();
Expand All @@ -278,23 +281,27 @@ public void parse(ParseContext context) throws IOException {
valuePreview = complexValue.toString();
}
} catch (Exception innerException) {
if (ignore_malformed == false) {
throw new MapperParsingException(
"failed to parse field [{}] of type [{}] in document with id '{}'. " + "Could not parse field value preview,",
e,
fieldType().name(),
fieldType().typeName(),
context.sourceToParse().id()
);
}
}

if (ignore_malformed == false) {
throw new MapperParsingException(
"failed to parse field [{}] of type [{}] in document with id '{}'. " + "Could not parse field value preview,",
"failed to parse field [{}] of type [{}] in document with id '{}'. " + "Preview of field's value: '{}'",
e,
fieldType().name(),
fieldType().typeName(),
context.sourceToParse().id()
context.sourceToParse().id(),
valuePreview
);
}

throw new MapperParsingException(
"failed to parse field [{}] of type [{}] in document with id '{}'. " + "Preview of field's value: '{}'",
e,
fieldType().name(),
fieldType().typeName(),
context.sourceToParse().id(),
valuePreview
);
}
multiFields.parse(this, context);
}
Expand Down

0 comments on commit 5e44c84

Please sign in to comment.