Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add nested and object fields to field capabilities response #33803

Merged
merged 6 commits into from
Sep 26, 2018
Merged

Add nested and object fields to field capabilities response #33803

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ee17abf
Add nested and object fields to field capabilities response
jimczi Sep 18, 2018
b47d6a9
fix test
jimczi Sep 18, 2018
3635231
Merge branch 'master' into field_caps_with_objects
jimczi Sep 19, 2018
235df36
apply review comments
jimczi Sep 25, 2018
0e8a40b
Merge branch 'master' into field_caps_with_objects
jimczi Sep 25, 2018
ab7879d
Merge branch 'master' into field_caps_with_objects
jimczi Sep 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 71 additions & 2 deletions rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,16 @@ setup:
nested2:
type: float
doc_values: false
level1:
type: nested
properties:
level2:
type: object
properties:
leaf1:
type: text
index: false

- do:
indices.create:
index: test2
Expand All @@ -48,6 +58,15 @@ setup:
nested2:
type: float
doc_values: true
level1:
type: nested
properties:
level2:
type: object
properties:
leaf1:
type: text
index: false
- do:
indices.create:
index: test3
Expand All @@ -64,14 +83,23 @@ setup:
geo:
type: keyword
object:
type: object
type: nested
properties:
nested1 :
type : long
index: false
nested2:
type: keyword
doc_values: false
level1:
type: object
properties:
level2:
type: object
properties:
leaf1:
type: text
index: false

---
"Get simple field caps":
Expand Down Expand Up @@ -112,7 +140,7 @@ setup:
- is_false: fields.geo.keyword.non_searchable_indices
- is_false: fields.geo.keyword.on_aggregatable_indices
---
"Get nested field caps":
"Get leaves field caps":

- do:
field_caps:
Expand Down Expand Up @@ -140,6 +168,47 @@ setup:
- is_false: fields.object\.nested2.keyword.non_aggregatable_indices
- is_false: fields.object\.nested2.keyword.non_searchable_indices
---
"Get object and nested field caps":
- skip:
version: " - 6.99.99"
reason: object and nested fields are returned since 7.0

- do:
field_caps:
index: 'test1,test2,test3'
fields: object*,level1*

- match: {fields.object.object.indices: ["test1", "test2"]}
- match: {fields.object.object.searchable: false}
- match: {fields.object.object.aggregatable: false}
- is_false: fields.object.object.non_aggregatable_indices
- is_false: fields.object.object.non_searchable_indices
- match: {fields.object.nested.indices: ["test3"]}
- match: {fields.object.nested.searchable: false}
- match: {fields.object.nested.aggregatable: false}
- is_false: fields.object.nested.non_aggregatable_indices
- is_false: fields.object.nested.non_searchable_indices
- match: {fields.level1.nested.indices: ["test1", "test2"]}
- match: {fields.level1.nested.searchable: false}
- match: {fields.level1.nested.aggregatable: false}
- is_false: fields.level1.nested.non_aggregatable_indices
- is_false: fields.level1.nested.non_searchable_indices
- match: {fields.level1.object.indices: ["test3"]}
- match: {fields.level1.object.searchable: false}
- match: {fields.level1.object.aggregatable: false}
- is_false: fields.level1.object.non_aggregatable_indices
- is_false: fields.level1.object.non_searchable_indices
- match: {fields.level1\.level2.object.searchable: false}
- match: {fields.level1\.level2.object.aggregatable: false}
- is_false: fields.level1\.level2.object.indices
- is_false: fields.level1\.level2.object.non_aggregatable_indices
- is_false: fields.level1\.level2.object.non_searchable_indices
- match: {fields.level1\.level2\.leaf1.text.searchable: false}
- match: {fields.level1\.level2\.leaf1.text.aggregatable: false}
- is_false: fields.level1\.level2\.leaf1.text.indices
- is_false: fields.level1\.level2\.leaf1.text.non_aggregatable_indices
- is_false: fields.level1\.level2\.leaf1.text..non_searchable_indices
---
"Get prefix field caps":

- do:
Expand Down
15 changes: 15 additions & 0 deletions ...c/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ObjectMapper;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.threadpool.ThreadPool;
Expand Down Expand Up @@ -86,6 +87,20 @@ protected FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesI
if (indicesService.isMetaDataField(field) || fieldPredicate.test(ft.name())) {
FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable());
responseMap.put(field, fieldCap);
} else {
continue;
}
// add nested and object fields
int dotIndex = ft.name().indexOf('.');
while (dotIndex > -1) {
String parentField = ft.name().substring(0, dotIndex);
ObjectMapper mapper = mapperService.getObjectMapper(parentField);
if (mapper != null) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was wondering whether this condition may ever be false, maybe mention sub fields in a comment? Should we also break as soon as responseMap.containsKey(parentField) is true?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also break as soon as responseMap.containsKey(parentField) is true?

If we start from the leaf, yes, I pushed
235df36 to reverse the iteration order.

String type = mapper.nested().isNested() ? "nested" : "object";
FieldCapabilities fieldCap = new FieldCapabilities(parentField, type, false, false);
responseMap.put(parentField, fieldCap);
}
dotIndex = ft.name().indexOf('.', dotIndex + 1);
}
}
}
Expand Down
33 changes: 23 additions & 10 deletions server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,12 @@
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.junit.Before;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
Expand Down Expand Up @@ -90,20 +93,26 @@ public void testGetFieldMappings() {
}

public void testFieldCapabilities() {
List<String> allFields = new ArrayList<>(ALL_FLAT_FIELDS);
allFields.addAll(ALL_OBJECT_FIELDS);
FieldCapabilitiesResponse index1 = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("index1")).actionGet();
assertFieldCaps(index1, ALL_FLAT_FIELDS);
assertFieldCaps(index1, allFields);
FieldCapabilitiesResponse filtered = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("filtered")).actionGet();
assertFieldCaps(filtered, FILTERED_FLAT_FIELDS);
List<String> filteredFields = new ArrayList<>(FILTERED_FLAT_FIELDS);
filteredFields.addAll(ALL_OBJECT_FIELDS);
assertFieldCaps(filtered, filteredFields);
//double check that submitting the filtered mappings to an unfiltered index leads to the same field_caps output
//as the one coming from a filtered index with same mappings
GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("filtered").get();
ImmutableOpenMap<String, MappingMetaData> filteredMapping = getMappingsResponse.getMappings().get("filtered");
assertAcked(client().admin().indices().prepareCreate("test").addMapping("_doc", filteredMapping.get("_doc").getSourceAsMap()));
FieldCapabilitiesResponse test = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("test")).actionGet();
assertFieldCaps(test, FILTERED_FLAT_FIELDS);
// properties.value is an object field in the new mapping
filteredFields.add("properties.value");
assertFieldCaps(test, filteredFields);
}

private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesResponse, String[] expectedFields) {
private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesResponse, Collection<String> expectedFields) {
Map<String, Map<String, FieldCapabilities>> responseMap = fieldCapabilitiesResponse.get();
Set<String> builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
for (String field : builtInMetaDataFields) {
Expand All @@ -118,7 +127,7 @@ private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesR
}

private static void assertFieldMappings(Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>> mappings,
String[] expectedFields) {
Collection<String> expectedFields) {
assertEquals(1, mappings.size());
Map<String, GetFieldMappingsResponse.FieldMappingMetaData> fields = new HashMap<>(mappings.get("_doc"));
Set<String> builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
Expand Down Expand Up @@ -245,14 +254,18 @@ public Function<String, Predicate<String>> getFieldFilter() {
}
}

private static final String[] ALL_FLAT_FIELDS = new String[]{
private static final Collection<String> ALL_FLAT_FIELDS = Arrays.asList(
"name.first", "name.last_visible", "birth", "age_visible", "address.street", "address.location", "address.area_visible",
"properties.key_visible", "properties.key_visible.keyword", "properties.value", "properties.value.keyword_visible"
};
);

private static final String[] FILTERED_FLAT_FIELDS = new String[]{
"name.last_visible", "age_visible", "address.area_visible", "properties.key_visible", "properties.value.keyword_visible"
};
private static final Collection<String> ALL_OBJECT_FIELDS = Arrays.asList(
"name", "address", "properties"
);

private static final Collection<String> FILTERED_FLAT_FIELDS = Arrays.asList(
"name.last_visible", "age_visible", "address.area_visible", "properties.key_visible", "properties.value.keyword_visible"
);

private static final String TEST_ITEM = "{\n" +
" \"_doc\": {\n" +
Expand Down