From 102611d1d3c8e25ffa40b481179c39ecd3e5bc62 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Fri, 28 Sep 2018 12:39:56 -0700 Subject: [PATCH] Add a simple JSON field mapper. (#33923) * Add a simple JSON field type. * Add support for ignore_above. * Add support for null_value. * Add support for split_queries_on_whitespace. * Prevent norms from being enabled. * Clarify the message around copy_to not being supported. * Disallow wildcard queries. * For now, disallow the field from being stored. --- .../index/mapper/JsonFieldMapper.java | 327 ++++++++++++++++ .../index/mapper/JsonFieldParser.java | 83 +++++ .../elasticsearch/indices/IndicesModule.java | 2 + .../index/mapper/JsonFieldMapperTests.java | 352 ++++++++++++++++++ .../index/mapper/JsonFieldParserTests.java | 157 ++++++++ .../index/mapper/JsonFieldTypeTests.java | 103 +++++ .../index/search/MultiMatchQueryTests.java | 36 ++ 7 files changed, 1060 insertions(+) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/JsonFieldTypeTests.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java new file mode 100644 index 0000000000000..7efcdf5ab9039 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java @@ -0,0 +1,327 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.analysis.AnalyzerScope; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.query.QueryShardContext; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.index.mapper.TypeParsers.parseField; + +/** + * A field mapper that accepts a JSON object and flattens it into a single field. This data type + * can be a useful alternative to an 'object' mapping when the object has a large, unknown set + * of keys. + * + * Currently the mapper extracts all leaf values of the JSON object, converts them to their text + * representations, and indexes each one as a keyword. As an example, given a json field called + * 'json_field' and the following input + * + * { + * "json_field: { + * "key1": "some value", + * "key2": { + * "key3": true + * } + * } + * } + * + * the mapper will produce untokenized string fields with the values "some value" and "true". + */ +public final class JsonFieldMapper extends FieldMapper { + + public static final String CONTENT_TYPE = "json"; + public static final NamedAnalyzer WHITESPACE_ANALYZER = new NamedAnalyzer( + "whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer()); + + private static class Defaults { + public static final MappedFieldType FIELD_TYPE = new JsonFieldType(); + + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setStored(false); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.freeze(); + } + + public static final int IGNORE_ABOVE = Integer.MAX_VALUE; + } + + public static class Builder extends FieldMapper.Builder { + private int ignoreAbove = Defaults.IGNORE_ABOVE; + + public Builder(String name) { + super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE); + builder = this; + } + + @Override + public JsonFieldType fieldType() { + return (JsonFieldType) super.fieldType(); + } + + @Override + public Builder indexOptions(IndexOptions indexOptions) { + if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) > 0) { + throw new IllegalArgumentException("The [" + CONTENT_TYPE + + "] field does not support positions, got [index_options]=" + + indexOptionToString(indexOptions)); + } + return super.indexOptions(indexOptions); + } + + public Builder ignoreAbove(int ignoreAbove) { + if (ignoreAbove < 0) { + throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove); + } + this.ignoreAbove = ignoreAbove; + return this; + } + + public Builder splitQueriesOnWhitespace(boolean splitQueriesOnWhitespace) { + fieldType().setSplitQueriesOnWhitespace(splitQueriesOnWhitespace); + return builder; + } + + @Override + public Builder addMultiField(Mapper.Builder mapperBuilder) { + throw new UnsupportedOperationException("[fields] is not supported for [" + CONTENT_TYPE + "] fields."); + } + + @Override + public Builder copyTo(CopyTo copyTo) { + throw new UnsupportedOperationException("[copy_to] is not supported for [" + CONTENT_TYPE + "] fields."); + } + + @Override + public Builder store(boolean store) { + throw new UnsupportedOperationException("[store] is not currently supported for [" + + CONTENT_TYPE + "] fields."); + } + + @Override + public JsonFieldMapper build(BuilderContext context) { + setupFieldType(context); + if (fieldType().splitQueriesOnWhitespace()) { + fieldType().setSearchAnalyzer(WHITESPACE_ANALYZER); + } + return new JsonFieldMapper(name, fieldType, defaultFieldType, + ignoreAbove, context.indexSettings()); + } + } + + public static class TypeParser implements Mapper.TypeParser { + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + JsonFieldMapper.Builder builder = new JsonFieldMapper.Builder(name); + parseField(builder, name, node, parserContext); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String propName = entry.getKey(); + Object propNode = entry.getValue(); + if (propName.equals("ignore_above")) { + builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1)); + iterator.remove(); + } else if (propName.equals("null_value")) { + if (propNode == null) { + throw new MapperParsingException("Property [null_value] cannot be null."); + } + builder.nullValue(propNode.toString()); + iterator.remove(); + } else if (propName.equals("split_queries_on_whitespace")) { + builder.splitQueriesOnWhitespace + (XContentMapValues.nodeBooleanValue(propNode, "split_queries_on_whitespace")); + iterator.remove(); + } + } + return builder; + } + } + + public static final class JsonFieldType extends StringFieldType { + private boolean splitQueriesOnWhitespace; + + public JsonFieldType() { + setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); + } + + private JsonFieldType(JsonFieldType ref) { + super(ref); + this.splitQueriesOnWhitespace = ref.splitQueriesOnWhitespace; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + JsonFieldType that = (JsonFieldType) o; + return splitQueriesOnWhitespace == that.splitQueriesOnWhitespace; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), splitQueriesOnWhitespace); + } + + public JsonFieldType clone() { + return new JsonFieldType(this); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + public boolean splitQueriesOnWhitespace() { + return splitQueriesOnWhitespace; + } + + public void setSplitQueriesOnWhitespace(boolean splitQueriesOnWhitespace) { + checkIfFrozen(); + this.splitQueriesOnWhitespace = splitQueriesOnWhitespace; + } + + @Override + public Query existsQuery(QueryShardContext context) { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + + @Override + public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, + boolean transpositions) { + throw new UnsupportedOperationException("[fuzzy] queries are not currently supported on [" + + CONTENT_TYPE + "] fields."); + } + + @Override + public Query regexpQuery(String value, int flags, int maxDeterminizedStates, + MultiTermQuery.RewriteMethod method, QueryShardContext context) { + throw new UnsupportedOperationException("[regexp] queries are not currently supported on [" + + CONTENT_TYPE + "] fields."); + } + + @Override + public Query wildcardQuery(String value, + MultiTermQuery.RewriteMethod method, + QueryShardContext context) { + throw new UnsupportedOperationException("[wildcard] queries are not currently supported on [" + + CONTENT_TYPE + "] fields."); + } + + @Override + public Object valueForDisplay(Object value) { + if (value == null) { + return null; + } + BytesRef binaryValue = (BytesRef) value; + return binaryValue.utf8ToString(); + } + } + + private final JsonFieldParser fieldParser; + private int ignoreAbove; + + private JsonFieldMapper(String simpleName, + MappedFieldType fieldType, + MappedFieldType defaultFieldType, + int ignoreAbove, + Settings indexSettings) { + super(simpleName, fieldType, defaultFieldType, indexSettings, MultiFields.empty(), CopyTo.empty()); + assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0; + + this.ignoreAbove = ignoreAbove; + this.fieldParser = new JsonFieldParser(fieldType, ignoreAbove); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + protected void doMerge(Mapper mergeWith) { + super.doMerge(mergeWith); + this.ignoreAbove = ((JsonFieldMapper) mergeWith).ignoreAbove; + } + + @Override + protected JsonFieldMapper clone() { + return (JsonFieldMapper) super.clone(); + } + + @Override + public JsonFieldType fieldType() { + return (JsonFieldType) super.fieldType(); + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { + return; + } + + if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { + fields.addAll(fieldParser.parse(context.parser())); + createFieldNamesField(context, fields); + } else { + context.parser().skipChildren(); + } + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + + if (includeDefaults || fieldType().nullValue() != null) { + builder.field("null_value", fieldType().nullValue()); + } + + if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) { + builder.field("ignore_above", ignoreAbove); + } + + if (includeDefaults || fieldType().splitQueriesOnWhitespace()) { + builder.field("split_queries_on_whitespace", fieldType().splitQueriesOnWhitespace()); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java new file mode 100644 index 0000000000000..25a40235844e9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * A helper class for {@link JsonFieldMapper} parses a JSON object + * and produces an indexable field for each leaf value. + */ +public class JsonFieldParser { + private final MappedFieldType fieldType; + private final int ignoreAbove; + + JsonFieldParser(MappedFieldType fieldType, + int ignoreAbove) { + this.fieldType = fieldType; + this.ignoreAbove = ignoreAbove; + } + + public List parse(XContentParser parser) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, + parser.currentToken(), + parser::getTokenLocation); + + List fields = new ArrayList<>(); + int openObjects = 1; + + while (true) { + if (openObjects == 0) { + return fields; + } + + XContentParser.Token token = parser.nextToken(); + assert token != null; + + if (token == XContentParser.Token.START_OBJECT) { + openObjects++; + } else if (token == XContentParser.Token.END_OBJECT) { + openObjects--; + } else if (token.isValue()) { + String value = parser.text(); + addField(value, fields); + } else if (token == XContentParser.Token.VALUE_NULL) { + String value = fieldType.nullValueAsString(); + if (value != null) { + addField(value, fields); + } + } + } + } + + private void addField(String value, List fields) { + if (value.length() <= ignoreAbove) { + fields.add(new Field(fieldType.name(), new BytesRef(value), fieldType)); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index bef05ecda9fd8..29d1b0f05d050 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -43,6 +43,7 @@ import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.JsonFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; @@ -133,6 +134,7 @@ public static Map getMappers(List mappe mappers.put(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser()); mappers.put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()); mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser()); + mappers.put(JsonFieldMapper.CONTENT_TYPE, new JsonFieldMapper.TypeParser()); mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); mappers.put(BaseGeoShapeFieldMapper.CONTENT_TYPE, new BaseGeoShapeFieldMapper.TypeParser()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java new file mode 100644 index 0000000000000..d15bc83068995 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java @@ -0,0 +1,352 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import org.junit.Before; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class JsonFieldMapperTests extends ESSingleNodeTestCase { + private IndexService indexService; + private DocumentMapperParser parser; + + @Before + public void setup() { + indexService = createIndex("test"); + parser = indexService.mapperService().documentMapperParser(); + } + + @Override + protected Collection> getPlugins() { + return pluginList(InternalSettingsPlugin.class); + } + + public void testDefaults() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .endObject() + .endObject() + .endObject() + .endObject()); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() + .startObject("field") + .field("key1", "value") + .field("key2", true) + .endObject() + .endObject()); + + ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON)); + IndexableField[] fields = parsedDoc.rootDoc().getFields("field"); + assertEquals(2, fields.length); + + IndexableField field1 = fields[0]; + assertEquals("field", field1.name()); + assertEquals(new BytesRef("value"), field1.binaryValue()); + assertTrue(field1.fieldType().omitNorms()); + + IndexableField field2 = fields[1]; + assertEquals("field", field2.name()); + assertEquals(new BytesRef("true"), field2.binaryValue()); + assertTrue(field2.fieldType().omitNorms()); + + IndexableField[] fieldNamesFields = parsedDoc.rootDoc().getFields(FieldNamesFieldMapper.NAME); + assertEquals(1, fieldNamesFields.length); + + IndexableField fieldNamesField = fieldNamesFields[0]; + assertEquals("field", fieldNamesField.stringValue()); + } + + public void testDisableIndex() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .field("index", false) + .endObject() + .endObject() + .endObject() + .endObject()); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() + .startObject("field") + .field("key", "value") + .endObject() + .endObject()); + + ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON)); + IndexableField[] fields = parsedDoc.rootDoc().getFields("field"); + assertEquals(0, fields.length); + } + + public void testEnableStore() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .field("store", true) + .endObject() + .endObject() + .endObject() + .endObject()); + + expectThrows(UnsupportedOperationException.class, () -> + parser.parse("type", new CompressedXContent(mapping))); + } + + public void testIndexOptions() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .field("index_options", "freqs") + .endObject() + .endObject() + .endObject() + .endObject()); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + for (String indexOptions : Arrays.asList("positions", "offsets")) { + String invalidMapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .field("index_options", indexOptions) + .endObject() + .endObject() + .endObject() + .endObject()); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> parser.parse("type", new CompressedXContent(invalidMapping))); + assertEquals("The [json] field does not support positions, got [index_options]=" + indexOptions, e.getMessage()); + } + } + + public void testNullField() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .endObject() + .endObject() + .endObject() + .endObject()); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() + .nullField("field") + .endObject()); + + ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON)); + IndexableField[] fields = parsedDoc.rootDoc().getFields("field"); + assertEquals(0, fields.length); + } + + public void testMalformedJson() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .endObject() + .endObject() + .endObject() + .endObject()); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + BytesReference doc1 = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() + .field("field", "not a JSON object") + .endObject()); + + expectThrows(MapperParsingException.class, () -> mapper.parse( + new SourceToParse("test", "type", "1", doc1, XContentType.JSON))); + + BytesReference doc2 = new BytesArray("{ \"field\": { \"key\": \"value\" "); + expectThrows(MapperParsingException.class, () -> mapper.parse( + new SourceToParse("test", "type", "1", doc2, XContentType.JSON))); + } + + public void testFieldMultiplicity() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .endObject() + .endObject() + .endObject() + .endObject()); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() + .startArray("field") + .startObject() + .field("key1", "value") + .endObject() + .startObject() + .field("key2", true) + .field("key3", false) + .endObject() + .endArray() + .endObject()); + + ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON)); + IndexableField[] fields = parsedDoc.rootDoc().getFields("field"); + assertEquals(3, fields.length); + + IndexableField field1 = fields[0]; + assertEquals("field", field1.name()); + assertEquals(new BytesRef("value"), field1.binaryValue()); + + IndexableField field2 = fields[1]; + assertEquals("field", field2.name()); + assertEquals(new BytesRef("true"), field2.binaryValue()); + + IndexableField field3 = fields[2]; + assertEquals("field", field3.name()); + assertEquals(new BytesRef("false"), field3.binaryValue()); + } + + public void testIgnoreAbove() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .field("ignore_above", 10) + .endObject() + .endObject() + .endObject() + .endObject()); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() + .startArray("field") + .startObject() + .field("key", "a longer than usual value") + .endObject() + .endArray() + .endObject()); + + ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON)); + IndexableField[] fields = parsedDoc.rootDoc().getFields("field"); + assertEquals(0, fields.length); + } + + + public void testNullValues() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .endObject() + .startObject("other_field") + .field("type", "json") + .field("null_value", "placeholder") + .endObject() + .endObject() + .endObject() + .endObject()); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() + .startObject("field") + .nullField("key") + .endObject() + .startObject("other_field") + .nullField("key") + .endObject() + .endObject()); + ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON)); + + IndexableField[] fields = parsedDoc.rootDoc().getFields("field"); + assertEquals(0, fields.length); + + IndexableField[] otherFields = parsedDoc.rootDoc().getFields("other_field"); + assertEquals(1, otherFields.length); + IndexableField field = otherFields[0]; + assertEquals(new BytesRef("placeholder"), field.binaryValue()); + } + + public void testSplitQueriesOnWhitespace() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .field("split_queries_on_whitespace", true) + .endObject() + .endObject() + .endObject().endObject()); + indexService.mapperService().merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + + MappedFieldType fieldType = indexService.mapperService().fullName("field"); + assertThat(fieldType, instanceOf(JsonFieldMapper.JsonFieldType.class)); + + JsonFieldMapper.JsonFieldType ft = (JsonFieldMapper.JsonFieldType) fieldType; + assertThat(ft.searchAnalyzer(), equalTo(JsonFieldMapper.WHITESPACE_ANALYZER)); + assertTokenStreamContents(ft.searchAnalyzer().analyzer().tokenStream("", "Hello World"), new String[] {"Hello", "World"}); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java new file mode 100644 index 0000000000000..a60637a87e14b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java @@ -0,0 +1,157 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.mapper.JsonFieldMapper.JsonFieldType; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +public class JsonFieldParserTests extends ESTestCase { + private JsonFieldParser parser; + + @Before + public void setUp() throws Exception { + super.setUp(); + + MappedFieldType fieldType = new JsonFieldType(); + fieldType.setName("field"); + parser = new JsonFieldParser(fieldType, Integer.MAX_VALUE); + } + + public void testTextValues() throws Exception { + String input = "{ \"key1\": \"value1\", \"key2\": \"value2\" }"; + XContentParser xContentParser = createXContentParser(input); + + List fields = parser.parse(xContentParser); + assertEquals(2, fields.size()); + + IndexableField field1 = fields.get(0); + assertEquals("field", field1.name()); + assertEquals(new BytesRef("value1"), field1.binaryValue()); + + IndexableField field2 = fields.get(1); + assertEquals("field", field2.name()); + assertEquals(new BytesRef("value2"), field2.binaryValue()); + } + + public void testNumericValues() throws Exception { + String input = "{ \"key\": 2.718 }"; + XContentParser xContentParser = createXContentParser(input); + + List fields = parser.parse(xContentParser); + assertEquals(1, fields.size()); + + IndexableField field = fields.get(0); + assertEquals("field", field.name()); + assertEquals(new BytesRef("2.718"), field.binaryValue()); + } + + public void testBooleanValues() throws Exception { + String input = "{ \"key\": false }"; + XContentParser xContentParser = createXContentParser(input); + + List fields = parser.parse(xContentParser); + assertEquals(1, fields.size()); + + IndexableField field = fields.get(0); + assertEquals("field", field.name()); + assertEquals(new BytesRef("false"), field.binaryValue()); + } + + public void testArrays() throws Exception { + String input = "{ \"key\": [true, false] }"; + XContentParser xContentParser = createXContentParser(input); + + List fields = parser.parse(xContentParser); + assertEquals(2, fields.size()); + + IndexableField field1 = fields.get(0); + assertEquals("field", field1.name()); + assertEquals(new BytesRef("true"), field1.binaryValue()); + + IndexableField field2 = fields.get(1); + assertEquals("field", field2.name()); + assertEquals(new BytesRef("false"), field2.binaryValue()); + } + + public void testNestedObjects() throws Exception { + String input = "{ \"parent1\": { \"key\" : \"value\" }," + + "\"parent2\": { \"key\" : \"value\" }}"; + XContentParser xContentParser = createXContentParser(input); + + List fields = parser.parse(xContentParser); + assertEquals(2, fields.size()); + + IndexableField field1 = fields.get(0); + assertEquals("field", field1.name()); + assertEquals(new BytesRef("value"), field1.binaryValue()); + + IndexableField field2 = fields.get(1); + assertEquals("field", field2.name()); + assertEquals(new BytesRef("value"), field2.binaryValue()); + } + + public void testIgnoreAbove() throws Exception { + String input = "{ \"key\": \"a longer field than usual\" }"; + XContentParser xContentParser = createXContentParser(input); + + JsonFieldType fieldType = new JsonFieldType(); + fieldType.setName("field"); + JsonFieldParser ignoreAboveParser = new JsonFieldParser(fieldType, 10); + + List fields = ignoreAboveParser.parse(xContentParser); + assertEquals(0, fields.size()); + } + + public void testNullValues() throws Exception { + String input = "{ \"key\": null}"; + XContentParser xContentParser = createXContentParser(input); + + List fields = parser.parse(xContentParser); + assertEquals(0, fields.size()); + + xContentParser = createXContentParser(input); + + JsonFieldType fieldType = new JsonFieldType(); + fieldType.setName("field"); + fieldType.setNullValue("placeholder"); + JsonFieldParser nullValueParser = new JsonFieldParser(fieldType, Integer.MAX_VALUE); + + fields = nullValueParser.parse(xContentParser); + assertEquals(1, fields.size()); + + IndexableField field = fields.get(0); + assertEquals("field", field.name()); + assertEquals(new BytesRef("placeholder"), field.binaryValue()); + } + + private XContentParser createXContentParser(String input) throws IOException { + XContentParser xContentParser = createParser(JsonXContent.jsonXContent, input); + xContentParser.nextToken(); + return xContentParser; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldTypeTests.java new file mode 100644 index 0000000000000..770e21a96d78d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldTypeTests.java @@ -0,0 +1,103 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.mapper.JsonFieldMapper.JsonFieldType; +import org.junit.Before; + +public class JsonFieldTypeTests extends FieldTypeTestCase { + + @Before + public void setupProperties() { + addModifier(new Modifier("split_queries_on_whitespace", true) { + @Override + public void modify(MappedFieldType type) { + JsonFieldType ft = (JsonFieldType) type; + ft.setSplitQueriesOnWhitespace(!ft.splitQueriesOnWhitespace()); + } + }); + } + + @Override + protected JsonFieldType createDefaultFieldType() { + return new JsonFieldType(); + } + + public void testValueForDisplay() { + JsonFieldType ft = createDefaultFieldType(); + ft.setName("field"); + + BytesRef indexedValue = ft.indexedValueForSearch("value"); + assertEquals("value", ft.valueForDisplay(indexedValue)); + } + + public void testTermQuery() { + JsonFieldType ft = createDefaultFieldType(); + ft.setName("field"); + + Query expected = new TermQuery(new Term("field", "value")); + assertEquals(expected, ft.termQuery("value", null)); + + ft.setIndexOptions(IndexOptions.NONE); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> ft.termQuery("field", null)); + assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + } + + public void testExistsQuery() { + JsonFieldType ft = createDefaultFieldType(); + ft.setName("field"); + + Query expected = new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))); + assertEquals(expected, ft.existsQuery(null)); + } + + public void testFuzzyQuery() { + JsonFieldType ft = createDefaultFieldType(); + ft.setName("field"); + + UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, + () -> ft.fuzzyQuery("valuee", Fuzziness.fromEdits(2), 1, 50, true)); + assertEquals("[fuzzy] queries are not currently supported on [json] fields.", e.getMessage()); + } + + public void testRegexpQuery() { + JsonFieldType ft = createDefaultFieldType(); + ft.setName("field"); + + UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, + () -> ft.regexpQuery("valu*", 0, 10, null, null)); + assertEquals("[regexp] queries are not currently supported on [json] fields.", e.getMessage()); + } + + public void testWildcardQuery() { + JsonFieldType ft = createDefaultFieldType(); + ft.setName("field"); + + UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, + () -> ft.wildcardQuery("valu*", null, null)); + assertEquals("[wildcard] queries are not currently supported on [json] fields.", e.getMessage()); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index 58baadd83573d..01b0ee5788bbb 100644 --- a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -335,4 +335,40 @@ public void testKeywordSplitQueriesOnWhitespace() throws IOException { ), 0.0f); assertThat(query, equalTo(expected)); } + + public void testJsonSplitQueriesOnWhitespace() throws IOException { + IndexService indexService = createIndex("test_json"); + MapperService mapperService = indexService.mapperService(); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .endObject() + .startObject("field_split") + .field("type", "json") + .field("split_queries_on_whitespace", true) + .endObject() + .endObject() + .endObject().endObject()); + mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + + QueryShardContext queryShardContext = indexService.newQueryShardContext(randomInt(20), + null, () -> { throw new UnsupportedOperationException(); }, null); + MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); + + Map fieldNames = new HashMap<>(); + fieldNames.put("field", 1.0f); + fieldNames.put("field_split", 1.0f); + Query query = parser.parse(MultiMatchQueryBuilder.Type.BEST_FIELDS, fieldNames, "Foo Bar", null); + + DisjunctionMaxQuery expected = new DisjunctionMaxQuery(Arrays.asList( + new TermQuery(new Term("field", "Foo Bar")), + new BooleanQuery.Builder() + .add(new TermQuery(new Term("field_split", "Foo")), BooleanClause.Occur.SHOULD) + .add(new TermQuery(new Term("field_split", "Bar")), BooleanClause.Occur.SHOULD) + .build() + ), 0.0f); + assertThat(query, equalTo(expected)); + } }