Skip to content

Commit

Permalink
Make sure non-collecting aggs include sub-aggs
Browse files Browse the repository at this point in the history
Now that we're consistently using `cat_match` to filter which shards we
run on we can get this confusing case:
1. You have a search with, say, a range and a sub-agg.
2. That search has a query that `can_match` can recognize will match no
   docs. On *any* shard.
3. So we dutifully run it on a single shard so it can produce the
   "empty" aggs.
4. The shard we pick happens to not have the target of the range mapped.
5. This kicks in the special range aggregator that doesn't collect any
   documents.
6. Before this commit, that range aggregator *also* never produced any
   sub-aggs.

So, without this change, it was quite possible for a search that
happened to match no documents to "throw away" the sub-aggs of a range
and a few other aggs.

We've had this problem for a long, long time but it is more confusing
now because `can_match` is really kicking in and causing us to see cases
where it looks like you are targeting a lot of shards but you really are
only targeting a couple. It used to be that to get the "no sub-aggs"
behavior you had to explicitly target only shards that didn't map the
target field of the `range` agg. And, like, in that case it isn't too
bad because you targeted a sort of degenerate shard. But now that
`can_match` is doing its thing you can end up with the confusing steps
above. It took me several hours to track down what what happening I know
how the individual pieces of all of this works. It took four hours to
figure out how they fit together in this case....

Anyway! This replaces all the aggregator implementations that throw out
the sub-aggregators with ones that keep them. I think this'll be less
confusing in the future.

Closes elastic#64142
  • Loading branch information
nik9000 committed Oct 27, 2020
1 parent 709264a commit 4bee8fe
Showing 12 changed files with 62 additions and 24 deletions.
Original file line number Diff line number Diff line change
@@ -60,7 +60,7 @@ public ChildrenAggregatorFactory(String name,

@Override
protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map<String, Object> metadata) throws IOException {
return new NonCollectingAggregator(name, searchContext, parent, metadata) {
return new NonCollectingAggregator(name, searchContext, parent, factories, metadata) {
@Override
public InternalAggregation buildEmptyAggregation() {
return new InternalChildren(name, 0, buildEmptySubAggregations(), metadata());
Original file line number Diff line number Diff line change
@@ -60,7 +60,7 @@ public ParentAggregatorFactory(String name,

@Override
protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map<String, Object> metadata) throws IOException {
return new NonCollectingAggregator(name, searchContext, parent, metadata) {
return new NonCollectingAggregator(name, searchContext, parent, factories, metadata) {
@Override
public InternalAggregation buildEmptyAggregation() {
return new InternalParent(name, 0, buildEmptySubAggregations(), metadata());
Original file line number Diff line number Diff line change
@@ -384,3 +384,39 @@ setup:
- match: { aggregations.age_groups.buckets.2.key: "Generation Y" }

- match: { aggregations.age_groups.buckets.2.doc_count: 2 }


---
"Date range unmapped with children":
- do:
indices.create:
index: test_a_unmapped
body:
settings:
number_of_shards: 1
number_of_replicas: 0
- do:
search:
index: test_a_unmapped
body:
size: 0
query:
terms:
animal: []
aggs:
date_range:
date_range:
field: date
ranges:
- from: 2020-01-01T00:00:00Z
aggs:
sounds:
cardinality:
field: sound.keyword

- match: { hits.total.value: 0 }
- length: { aggregations.date_range.buckets: 1 }
- match: { aggregations.date_range.buckets.0.doc_count: 0 }
- match: { aggregations.date_range.buckets.0.key: "2020-01-01T00:00:00.000Z-*" }
- is_false: aggregations.date_range.buckets.0.to
- match: { aggregations.date_range.buckets.0.sounds.value: 0 }
Original file line number Diff line number Diff line change
@@ -38,14 +38,6 @@ protected NonCollectingAggregator(String name, SearchContext context, Aggregator
super(name, subFactories, context, parent, CardinalityUpperBound.NONE, metadata);
}

/**
* Build a {@linkplain NonCollectingAggregator} for an aggregator without sub-aggregators.
*/
protected NonCollectingAggregator(String name, SearchContext context, Aggregator parent,
Map<String, Object> metadata) throws IOException {
this(name, context, parent, AggregatorFactories.EMPTY, metadata);
}

@Override
public final LeafBucketCollector getLeafCollector(LeafReaderContext reader, LeafBucketCollector sub) {
// the framework will automatically eliminate it
Original file line number Diff line number Diff line change
@@ -63,7 +63,7 @@ protected Aggregator createUnmapped(SearchContext searchContext,
Aggregator parent,
Map<String, Object> metadata) throws IOException {
final InternalAggregation aggregation = new InternalGeoHashGrid(name, requiredSize, emptyList(), metadata);
return new NonCollectingAggregator(name, searchContext, parent, metadata) {
return new NonCollectingAggregator(name, searchContext, parent, factories, metadata) {
@Override
public InternalAggregation buildEmptyAggregation() {
return aggregation;
Original file line number Diff line number Diff line change
@@ -61,7 +61,7 @@ protected Aggregator createUnmapped(SearchContext searchContext,
Aggregator parent,
Map<String, Object> metadata) throws IOException {
final InternalAggregation aggregation = new InternalGeoTileGrid(name, requiredSize, Collections.emptyList(), metadata);
return new NonCollectingAggregator(name, searchContext, parent, metadata) {
return new NonCollectingAggregator(name, searchContext, parent, factories, metadata) {
@Override
public InternalAggregation buildEmptyAggregation() {
return aggregation;
Original file line number Diff line number Diff line change
@@ -51,7 +51,7 @@ public Aggregator createInternal(SearchContext searchContext,
CardinalityUpperBound cardinality,
Map<String, Object> metadata) throws IOException {
if (childObjectMapper == null) {
return new Unmapped(name, searchContext, parent, metadata);
return new Unmapped(name, searchContext, parent, factories, metadata);
}
return new NestedAggregator(name, factories, parentObjectMapper, childObjectMapper, searchContext, parent,
cardinality, metadata);
@@ -62,8 +62,9 @@ private static final class Unmapped extends NonCollectingAggregator {
Unmapped(String name,
SearchContext context,
Aggregator parent,
AggregatorFactories factories,
Map<String, Object> metadata) throws IOException {
super(name, context, parent, metadata);
super(name, context, parent, factories, metadata);
}

@Override
Original file line number Diff line number Diff line change
@@ -52,7 +52,7 @@ public Aggregator createInternal(SearchContext searchContext,
CardinalityUpperBound cardinality,
Map<String, Object> metadata) throws IOException {
if (unmapped) {
return new Unmapped(name, searchContext, parent, metadata);
return new Unmapped(name, searchContext, parent, factories, metadata);
} else {
return new ReverseNestedAggregator(name, factories, parentObjectMapper,
searchContext, parent, cardinality, metadata);
@@ -64,8 +64,9 @@ private static final class Unmapped extends NonCollectingAggregator {
Unmapped(String name,
SearchContext context,
Aggregator parent,
AggregatorFactories factories,
Map<String, Object> metadata) throws IOException {
super(name, context, parent, metadata);
super(name, context, parent, factories, metadata);
}

@Override
Original file line number Diff line number Diff line change
@@ -76,7 +76,7 @@ public AbstractRangeAggregatorFactory(String name,
protected Aggregator createUnmapped(SearchContext searchContext,
Aggregator parent,
Map<String, Object> metadata) throws IOException {
return new Unmapped<>(name, ranges, keyed, config.format(), searchContext, parent, rangeFactory, metadata);
return new Unmapped<>(name, factories, ranges, keyed, config.format(), searchContext, parent, rangeFactory, metadata);
}

@Override
Original file line number Diff line number Diff line change
@@ -107,7 +107,7 @@ public GeoDistanceRangeAggregatorFactory(String name, ValuesSourceConfig config,
protected Aggregator createUnmapped(SearchContext searchContext,
Aggregator parent,
Map<String, Object> metadata) throws IOException {
return new RangeAggregator.Unmapped<>(name, ranges, keyed, config.format(), searchContext, parent,
return new RangeAggregator.Unmapped<>(name, factories, ranges, keyed, config.format(), searchContext, parent,
rangeFactory, metadata);
}

Original file line number Diff line number Diff line change
@@ -353,11 +353,19 @@ public static class Unmapped<R extends RangeAggregator.Range> extends NonCollect
private final InternalRange.Factory factory;
private final DocValueFormat format;

public Unmapped(String name, R[] ranges, boolean keyed, DocValueFormat format, SearchContext context, Aggregator parent,
InternalRange.Factory factory, Map<String, Object> metadata)
throws IOException {

super(name, context, parent, metadata);
public Unmapped(
String name,
AggregatorFactories factories,
R[] ranges,
boolean keyed,
DocValueFormat format,
SearchContext context,
Aggregator parent,
InternalRange.Factory factory,
Map<String, Object> metadata
) throws IOException {

super(name, context, parent, factories, metadata);
this.ranges = ranges;
this.keyed = keyed;
this.format = format;
Original file line number Diff line number Diff line change
@@ -186,7 +186,7 @@ protected Aggregator createUnmapped(SearchContext searchContext,
Map<String, Object> metadata) throws IOException {
final InternalAggregation aggregation = new UnmappedSignificantTerms(name, bucketCountThresholds.getRequiredSize(),
bucketCountThresholds.getMinDocCount(), metadata);
return new NonCollectingAggregator(name, searchContext, parent, metadata) {
return new NonCollectingAggregator(name, searchContext, parent, factories, metadata) {
@Override
public InternalAggregation buildEmptyAggregation() {
return aggregation;

0 comments on commit 4bee8fe

Please sign in to comment.