Skip to content

Commit

Permalink
don't account _source.mode mapping attributes usages
Browse files Browse the repository at this point in the history
  • Loading branch information
martijnvg committed Nov 27, 2024
1 parent 1ddd071 commit 9cf03fd
Show file tree
Hide file tree
Showing 3 changed files with 2 additions and 95 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -48,56 +48,3 @@ test source modes:
- match: { indices.mappings.source_modes.disabled: 1 }
- match: { indices.mappings.source_modes.stored: 1 }
- match: { indices.mappings.source_modes.synthetic: 1 }

---
test old mapping source modes:
- requires:
cluster_features: [ "cluster.stats.source_modes" ]
reason: requires source modes features

- do:
indices.create:
index: test-synthetic
body:
mappings:
_source:
mode: synthetic

- do:
indices.create:
index: test-stored
body:
mappings:
_source:
mode: stored

- do:
indices.create:
index: test-disabled
body:
mappings:
_source:
mode: disabled

- do:
bulk:
refresh: true
body:
- '{ "create": { "_index": "test-synthetic" } }'
- '{ "name": "aaaa", "some_string": "AaAa", "some_int": 1000, "some_double": 123.456789, "some_bool": true }'
- '{ "create": { "_index": "test-stored" } }'
- '{ "name": "bbbb", "some_string": "BbBb", "some_int": 2000, "some_double": 321.987654, "some_bool": false }'
- '{ "create": { "_index": "test-disabled" } }'
- '{ "name": "cccc", "some_string": "CcCc", "some_int": 3000, "some_double": 421.484654, "some_bool": false }'

- do:
search:
index: test-*
- match: { hits.total.value: 3 }

- do:
cluster.stats: { }

- match: { indices.mappings.source_modes.disabled: 1 }
- match: { indices.mappings.source_modes.stored: 1 }
- match: { indices.mappings.source_modes.synthetic: 1 }
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
Map<String, FieldStats> fieldTypes = new HashMap<>();
Set<String> concreteFieldNames = new HashSet<>();
// Account different source modes based on index.mapping.source.mode setting:
Map<String, Integer> indexSourceModeUsageCount = new HashMap<>();
Map<String, Integer> sourceModeUsageCount = new HashMap<>();
Map<String, RuntimeFieldStats> runtimeFieldTypes = new HashMap<>();
final Map<MappingMetadata, Integer> mappingCounts = new IdentityHashMap<>(metadata.getMappingsByHash().size());
for (IndexMetadata indexMetadata : metadata) {
Expand All @@ -71,26 +71,16 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
AnalysisStats.countMapping(mappingCounts, indexMetadata);

var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings());
indexSourceModeUsageCount.compute(sourceMode.toString().toLowerCase(Locale.ENGLISH), (k, v) -> v == null ? 1 : v + 1);
sourceModeUsageCount.compute(sourceMode.toString().toLowerCase(Locale.ENGLISH), (k, v) -> v == null ? 1 : v + 1);
}
final AtomicLong totalFieldCount = new AtomicLong();
final AtomicLong totalDeduplicatedFieldCount = new AtomicLong();
// Account different source modes based on _source.mode mapping attribute in a separate map:
// (Materializing the mapping into map of maps is expensive and only happens here once per unique mapping)
Map<String, Integer> mappingSourceModeUsageCount = new HashMap<>();
for (Map.Entry<MappingMetadata, Integer> mappingAndCount : mappingCounts.entrySet()) {
ensureNotCancelled.run();
Set<String> indexFieldTypes = new HashSet<>();
Set<String> indexRuntimeFieldTypes = new HashSet<>();
final int count = mappingAndCount.getValue();
final Map<String, Object> map = mappingAndCount.getKey().getSourceAsMap();
if (map.containsKey("_source")) {
Map<?, ?> sourceFieldDefinition = (Map<?, ?>) map.get("_source");
if (sourceFieldDefinition.containsKey("mode")) {
String mode = (String) sourceFieldDefinition.get("mode");
mappingSourceModeUsageCount.compute(mode.toString().toLowerCase(Locale.ENGLISH), (k, v) -> v == null ? 1 : v + 1);
}
}
MappingVisitor.visitMapping(map, (field, fieldMapping) -> {
concreteFieldNames.add(field);
String type = null;
Expand Down Expand Up @@ -196,10 +186,6 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
totalMappingSizeBytes += mappingMetadata.source().compressed().length;
}

// Either pick counts based on the index setting or mapping attribute. The counts can't be combined otherwise modes get over-counted
// (The mappingSourceModeUsageCount is under-counted because duplicate mappings are skipped. This is at least gives some insight in
// case old _source.mode mapping attribute is used)
var sourceModeUsageCount = mappingSourceModeUsageCount.isEmpty() == false ? mappingSourceModeUsageCount : indexSourceModeUsageCount;
return new MappingStats(
totalFieldCount.get(),
totalDeduplicatedFieldCount.get(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
import static org.elasticsearch.index.mapper.SourceFieldMapper.Mode.STORED;
import static org.elasticsearch.index.mapper.SourceFieldMapper.Mode.SYNTHETIC;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;

public class MappingStatsTests extends AbstractWireSerializingTestCase<MappingStats> {

Expand Down Expand Up @@ -614,29 +613,4 @@ public void testSourceModes() {
assertThat(mappingStats.getSourceModeUsageCount().get("disabled"), equalTo(numDisabledIndices));
}

public void testSourceModesOldSourceMode() {
var builder = Metadata.builder();
String mapping = """
{
"_source": {
"mode": "synthetic"
}
}
""";
var indexMetadata1 = new IndexMetadata.Builder("foo-synthetic").settings(indexSettings(IndexVersion.current(), 4, 1))
.putMapping(mapping);
var indexMetadata2 = new IndexMetadata.Builder("foo-stored").settings(
indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "stored")
);
var indexMetadata3 = new IndexMetadata.Builder("foo-disabled").settings(
indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "disabled")
);
builder.put(indexMetadata1);
builder.put(indexMetadata2);
builder.put(indexMetadata3);
var mappingStats = MappingStats.of(builder.build(), () -> {});
assertThat(mappingStats.getSourceModeUsageCount().get("synthetic"), equalTo(1));
assertThat(mappingStats.getSourceModeUsageCount().get("stored"), nullValue());
assertThat(mappingStats.getSourceModeUsageCount().get("disabled"), nullValue());
}
}

0 comments on commit 9cf03fd

Please sign in to comment.