diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java index 1e77b4c2b002f..62cbeebeafe84 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; @@ -28,11 +29,14 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -71,6 +75,9 @@ * * Note that \0 is a reserved separator character, and cannot be used in the keys of the JSON object * (see {@link JsonFieldParser#SEPARATOR}). + * + * When 'store' is enabled, a single stored field is added containing the entire JSON object in + * pretty-printed format. */ public final class JsonFieldMapper extends FieldMapper { @@ -139,12 +146,6 @@ public Builder copyTo(CopyTo copyTo) { throw new UnsupportedOperationException("[copy_to] is not supported for [" + CONTENT_TYPE + "] fields."); } - @Override - public Builder store(boolean store) { - throw new UnsupportedOperationException("[store] is not currently supported for [" + - CONTENT_TYPE + "] fields."); - } - @Override public JsonFieldMapper build(BuilderContext context) { setupFieldType(context); @@ -377,7 +378,8 @@ private JsonFieldMapper(String simpleName, assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0; this.ignoreAbove = ignoreAbove; - this.fieldParser = new JsonFieldParser(fieldType.name(), keyedFieldName(), fieldType, ignoreAbove); + this.fieldParser = new JsonFieldParser(fieldType.name(), keyedFieldName(), + ignoreAbove, fieldType.nullValueAsString()); } @Override @@ -415,12 +417,36 @@ protected void parseCreateField(ParseContext context, List field return; } - if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { - fields.addAll(fieldParser.parse(context.parser())); - createFieldNamesField(context, fields); - } else { + if (fieldType.indexOptions() == IndexOptions.NONE && !fieldType.stored()) { context.parser().skipChildren(); + return; + } + + BytesRef storedValue = null; + if (fieldType.stored()) { + XContentBuilder builder = XContentFactory.jsonBuilder() + .prettyPrint() + .copyCurrentStructure(context.parser()); + storedValue = BytesReference.bytes(builder).toBytesRef(); + fields.add(new StoredField(fieldType.name(), storedValue)); } + + if (fieldType().indexOptions() != IndexOptions.NONE) { + XContentParser indexedFieldsParser = context.parser(); + + // If store is enabled, we've already consumed the content to produce the stored field. Here we + // 'reset' the parser, so that we can traverse the content again. + if (storedValue != null) { + indexedFieldsParser = JsonXContent.jsonXContent.createParser(context.parser().getXContentRegistry(), + context.parser().getDeprecationHandler(), + storedValue.bytes); + indexedFieldsParser.nextToken(); + } + + fields.addAll(fieldParser.parse(indexedFieldsParser)); + } + + createFieldNamesField(context, fields); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java index abd481d0f0ad7..fd05c07538c5b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.xcontent.XContentParser; @@ -39,17 +40,17 @@ public class JsonFieldParser { private final String rootFieldName; private final String keyedFieldName; - private final MappedFieldType fieldType; private final int ignoreAbove; + private final String nullValueAsString; JsonFieldParser(String rootFieldName, String keyedFieldName, - MappedFieldType fieldType, - int ignoreAbove) { + int ignoreAbove, + String nullValueAsString) { this.rootFieldName = rootFieldName; this.keyedFieldName = keyedFieldName; - this.fieldType = fieldType; this.ignoreAbove = ignoreAbove; + this.nullValueAsString = nullValueAsString; } public List parse(XContentParser parser) throws IOException { @@ -111,9 +112,8 @@ private void parseFieldValue(XContentParser.Token token, String value = parser.text(); addField(path, currentName, value, fields); } else if (token == XContentParser.Token.VALUE_NULL) { - String value = fieldType.nullValueAsString(); - if (value != null) { - addField(path, currentName, value, fields); + if (nullValueAsString != null) { + addField(path, currentName, nullValueAsString, fields); } } else { // Note that we throw an exception here just to be safe. We don't actually expect to reach @@ -137,8 +137,8 @@ private void addField(ContentPath path, } String keyedValue = createKeyedValue(key, value); - fields.add(new Field(rootFieldName, new BytesRef(value), fieldType)); - fields.add(new Field(keyedFieldName, new BytesRef(keyedValue), fieldType)); + fields.add(new StringField(rootFieldName, new BytesRef(value), Field.Store.NO)); + fields.add(new StringField(keyedFieldName, new BytesRef(keyedValue), Field.Store.NO)); } public static String createKeyedValue(String key, String value) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java index fc3c529ba802e..f089052e61f2e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.JsonFieldMapper.RootJsonFieldType; import org.elasticsearch.plugins.Plugin; @@ -130,16 +131,51 @@ public void testEnableStore() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() .startObject("type") .startObject("properties") - .startObject("field") + .startObject("store_and_index") .field("type", "json") .field("store", true) .endObject() + .startObject("store_only") + .field("type", "json") + .field("index", false) + .field("store", true) + .endObject() .endObject() .endObject() .endObject()); - expectThrows(UnsupportedOperationException.class, () -> - parser.parse("type", new CompressedXContent(mapping))); + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() + .startObject("store_only") + .field("key", "value") + .endObject() + .startObject("store_and_index") + .field("key", "value") + .endObject() + .endObject()); + ParsedDocument parsedDoc = mapper.parse(SourceToParse.source("test", "type", "1", doc, XContentType.JSON)); + + // We make sure to pretty-print here, since the field is always stored in pretty-printed format. + BytesReference storedValue = BytesReference.bytes(JsonXContent.contentBuilder() + .prettyPrint() + .startObject() + .field("key", "value") + .endObject()); + + IndexableField[] storeOnly = parsedDoc.rootDoc().getFields("store_only"); + assertEquals(1, storeOnly.length); + + assertTrue(storeOnly[0].fieldType().stored()); + assertEquals(storedValue.toBytesRef(), storeOnly[0].binaryValue()); + + IndexableField[] storeAndIndex = parsedDoc.rootDoc().getFields("store_and_index"); + assertEquals(2, storeAndIndex.length); + + assertTrue(storeAndIndex[0].fieldType().stored()); + assertEquals(storedValue.toBytesRef(), storeAndIndex[0].binaryValue()); + assertFalse(storeAndIndex[1].fieldType().stored()); } public void testIndexOptions() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java index fa6d7e878274b..715397167fd75 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java @@ -41,10 +41,7 @@ public class JsonFieldParserTests extends ESTestCase { @Before public void setUp() throws Exception { super.setUp(); - - MappedFieldType fieldType = new RootJsonFieldType(); - fieldType.setName("field"); - parser = new JsonFieldParser("field", "field._keyed", fieldType, Integer.MAX_VALUE); + parser = new JsonFieldParser("field", "field._keyed", Integer.MAX_VALUE, null); } public void testTextValues() throws Exception { @@ -222,9 +219,9 @@ public void testIgnoreAbove() throws Exception { RootJsonFieldType fieldType = new RootJsonFieldType(); fieldType.setName("field"); - JsonFieldParser ignoreAboveParser = new JsonFieldParser("field", "field._keyed", fieldType, 10); + JsonFieldParser parserWithIgnoreAbove = new JsonFieldParser("field", "field._keyed", 10, null); - List fields = ignoreAboveParser.parse(xContentParser); + List fields = parserWithIgnoreAbove.parse(xContentParser); assertEquals(0, fields.size()); } @@ -236,13 +233,10 @@ public void testNullValues() throws Exception { assertEquals(0, fields.size()); xContentParser = createXContentParser(input); + JsonFieldParser parserWithNullValue = new JsonFieldParser("field", "field._keyed", + Integer.MAX_VALUE, "placeholder"); - RootJsonFieldType fieldType = new RootJsonFieldType(); - fieldType.setName("field"); - fieldType.setNullValue("placeholder"); - JsonFieldParser nullValueParser = new JsonFieldParser("field", "field._keyed", fieldType, Integer.MAX_VALUE); - - fields = nullValueParser.parse(xContentParser); + fields = parserWithNullValue.parse(xContentParser); assertEquals(2, fields.size()); IndexableField field = fields.get(0);