diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 736010996aa1a..39d14f84145bc 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -42,6 +42,7 @@ import org.elasticsearch.aggregations.bucket.adjacency.ParsedAdjacencyMatrix; import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; import org.elasticsearch.aggregations.bucket.histogram.ParsedAutoDateHistogram; +import org.elasticsearch.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.client.analytics.ParsedStringStats; import org.elasticsearch.client.analytics.ParsedTopMetrics; import org.elasticsearch.client.analytics.StringStatsAggregationBuilder; @@ -151,7 +152,6 @@ import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.WeightedAvgAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.InternalBucketMetricValue; import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/BucketSelectorIT.java similarity index 98% rename from server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java rename to modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/BucketSelectorIT.java index 18981ef0ba813..534f515ddb777 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/BucketSelectorIT.java @@ -6,10 +6,11 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.pipeline; +package org.elasticsearch.aggregations.pipeline; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.aggregations.AggregationsPlugin; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptPlugin; @@ -35,7 +36,6 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.bucketSelector; -import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.derivative; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; @@ -60,7 +60,7 @@ public class BucketSelectorIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singleton(CustomScriptPlugin.class); + return List.of(CustomScriptPlugin.class, AggregationsPlugin.class); } public static class CustomScriptPlugin extends MockScriptPlugin { @@ -570,7 +570,9 @@ public void testEmptyBuckets() { .interval(1) .extendedBounds(1L, 4L) .minDocCount(0) - .subAggregation(derivative("derivative", "_count").gapPolicy(GapPolicy.INSERT_ZEROS)) + .subAggregation( + new DerivativePipelineAggregationBuilder("derivative", "_count").gapPolicy(GapPolicy.INSERT_ZEROS) + ) ) ) .get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java similarity index 95% rename from server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java rename to modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java index 88626c288480d..628863387b157 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java @@ -6,18 +6,21 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.pipeline; +package org.elasticsearch.aggregations.pipeline; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.aggregations.AggregationsPlugin; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateFormatters; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; import org.elasticsearch.search.aggregations.metrics.Sum; +import org.elasticsearch.search.aggregations.pipeline.SimpleValue; import org.elasticsearch.search.aggregations.support.AggregationPath; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matcher; @@ -31,11 +34,11 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; -import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.derivative; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.closeTo; @@ -56,6 +59,11 @@ private ZonedDateTime date(int month, int day) { return ZonedDateTime.of(2012, month, day, 0, 0, 0, 0, ZoneOffset.UTC); } + @Override + protected Collection> nodePlugins() { + return List.of(AggregationsPlugin.class); + } + private static IndexRequestBuilder indexDoc(String idx, ZonedDateTime date, int value) throws Exception { return client().prepareIndex(idx).setSource(jsonBuilder().startObject().timeField("date", date).field("value", value).endObject()); } @@ -113,7 +121,7 @@ public void testSingleValuedField() throws Exception { dateHistogram("histo").field("date") .calendarInterval(DateHistogramInterval.MONTH) .minDocCount(0) - .subAggregation(derivative("deriv", "_count")) + .subAggregation(new DerivativePipelineAggregationBuilder("deriv", "_count")) ) .get(); @@ -158,7 +166,7 @@ public void testSingleValuedFieldNormalised() throws Exception { dateHistogram("histo").field("date") .calendarInterval(DateHistogramInterval.MONTH) .minDocCount(0) - .subAggregation(derivative("deriv", "_count").unit(DateHistogramInterval.DAY)) + .subAggregation(new DerivativePipelineAggregationBuilder("deriv", "_count").unit(DateHistogramInterval.DAY)) ) .get(); @@ -223,7 +231,7 @@ public void testSingleValuedFieldNormalised_timeZone_CET_DstStart() throws Excep .calendarInterval(DateHistogramInterval.DAY) .timeZone(timezone) .minDocCount(0) - .subAggregation(derivative("deriv", "_count").unit(DateHistogramInterval.HOUR)) + .subAggregation(new DerivativePipelineAggregationBuilder("deriv", "_count").unit(DateHistogramInterval.HOUR)) ) .get(); @@ -281,7 +289,7 @@ public void testSingleValuedFieldNormalised_timeZone_CET_DstEnd() throws Excepti .calendarInterval(DateHistogramInterval.DAY) .timeZone(timezone) .minDocCount(0) - .subAggregation(derivative("deriv", "_count").unit(DateHistogramInterval.HOUR)) + .subAggregation(new DerivativePipelineAggregationBuilder("deriv", "_count").unit(DateHistogramInterval.HOUR)) ) .get(); @@ -341,7 +349,7 @@ public void testSingleValuedFieldNormalised_timeZone_AsiaKathmandu() throws Exce .calendarInterval(DateHistogramInterval.HOUR) .timeZone(timezone) .minDocCount(0) - .subAggregation(derivative("deriv", "_count").unit(DateHistogramInterval.MINUTE)) + .subAggregation(new DerivativePipelineAggregationBuilder("deriv", "_count").unit(DateHistogramInterval.MINUTE)) ) .get(); @@ -409,7 +417,7 @@ public void testSingleValuedFieldWithSubAggregation() throws Exception { .calendarInterval(DateHistogramInterval.MONTH) .minDocCount(0) .subAggregation(sum("sum").field("value")) - .subAggregation(derivative("deriv", "sum")) + .subAggregation(new DerivativePipelineAggregationBuilder("deriv", "sum")) ) .get(); @@ -492,7 +500,7 @@ public void testMultiValuedField() throws Exception { dateHistogram("histo").field("dates") .calendarInterval(DateHistogramInterval.MONTH) .minDocCount(0) - .subAggregation(derivative("deriv", "_count")) + .subAggregation(new DerivativePipelineAggregationBuilder("deriv", "_count")) ) .get(); @@ -550,7 +558,7 @@ public void testUnmapped() throws Exception { dateHistogram("histo").field("date") .calendarInterval(DateHistogramInterval.MONTH) .minDocCount(0) - .subAggregation(derivative("deriv", "_count")) + .subAggregation(new DerivativePipelineAggregationBuilder("deriv", "_count")) ) .get(); @@ -568,7 +576,7 @@ public void testPartiallyUnmapped() throws Exception { dateHistogram("histo").field("date") .calendarInterval(DateHistogramInterval.MONTH) .minDocCount(0) - .subAggregation(derivative("deriv", "_count")) + .subAggregation(new DerivativePipelineAggregationBuilder("deriv", "_count")) ) .get(); diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java index 1553470a47403..007c22920b15e 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java @@ -12,6 +12,8 @@ import org.elasticsearch.aggregations.bucket.adjacency.InternalAdjacencyMatrix; import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; import org.elasticsearch.aggregations.bucket.histogram.InternalAutoDateHistogram; +import org.elasticsearch.aggregations.pipeline.Derivative; +import org.elasticsearch.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.aggregations.pipeline.MovFnPipelineAggregationBuilder; import org.elasticsearch.aggregations.pipeline.MovingFunctionScript; import org.elasticsearch.plugins.Plugin; @@ -42,6 +44,11 @@ public List getAggregations() { @Override public List getPipelineAggregations() { return List.of( + new PipelineAggregationSpec( + DerivativePipelineAggregationBuilder.NAME, + DerivativePipelineAggregationBuilder::new, + DerivativePipelineAggregationBuilder::parse + ).addResultReader(Derivative::new), new PipelineAggregationSpec( MovFnPipelineAggregationBuilder.NAME, MovFnPipelineAggregationBuilder::new, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalDerivative.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/Derivative.java similarity index 83% rename from server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalDerivative.java rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/Derivative.java index 14a790b079dc6..253b416d97f04 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalDerivative.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/Derivative.java @@ -6,11 +6,12 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.pipeline; +package org.elasticsearch.aggregations.pipeline; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -18,10 +19,10 @@ import java.util.Map; import java.util.Objects; -public class InternalDerivative extends InternalSimpleValue implements Derivative { +public class Derivative extends InternalSimpleValue { private final double normalizationFactor; - InternalDerivative(String name, double value, double normalizationFactor, DocValueFormat formatter, Map metadata) { + Derivative(String name, double value, double normalizationFactor, DocValueFormat formatter, Map metadata) { super(name, value, formatter, metadata); this.normalizationFactor = normalizationFactor; } @@ -29,7 +30,7 @@ public class InternalDerivative extends InternalSimpleValue implements Derivativ /** * Read from a stream. */ - public InternalDerivative(StreamInput in) throws IOException { + public Derivative(StreamInput in) throws IOException { super(in); normalizationFactor = in.readDouble(); } @@ -45,7 +46,12 @@ public String getWriteableName() { return DerivativePipelineAggregationBuilder.NAME; } - @Override + /** + * Returns the normalized value. If no normalised factor has been specified + * this method will return {@link #value()} + * + * @return the normalized value + */ public double normalizedValue() { return normalizationFactor > 0 ? (value() / normalizationFactor) : value(); } @@ -95,7 +101,7 @@ public boolean equals(Object obj) { if (this == obj) return true; if (obj == null || getClass() != obj.getClass()) return false; if (super.equals(obj) == false) return false; - InternalDerivative other = (InternalDerivative) obj; + Derivative other = (Derivative) obj; return Objects.equals(value, other.value) && Objects.equals(normalizationFactor, other.normalizationFactor); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregationBuilder.java similarity index 97% rename from server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregationBuilder.java index 5ddfac0d22e1b..f4838441938ee 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregationBuilder.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.pipeline; +package org.elasticsearch.aggregations.pipeline; import org.elasticsearch.Version; import org.elasticsearch.common.ParsingException; @@ -17,7 +17,9 @@ import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregator.java similarity index 94% rename from server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregator.java rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregator.java index 21bed0e9f273f..89d445903f8cc 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregator.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.pipeline; +package org.elasticsearch.aggregations.pipeline; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationReduceContext; @@ -16,6 +16,7 @@ import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket; import org.elasticsearch.search.aggregations.bucket.histogram.HistogramFactory; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import java.util.ArrayList; import java.util.List; @@ -70,7 +71,7 @@ public InternalAggregation reduce(InternalAggregation aggregation, AggregationRe final List aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false) .map((p) -> (InternalAggregation) p) .collect(Collectors.toCollection(ArrayList::new)); - aggs.add(new InternalDerivative(name(), gradient, xDiff, formatter, metadata())); + aggs.add(new Derivative(name(), gradient, xDiff, formatter, metadata())); Bucket newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), InternalAggregations.from(aggs)); newBuckets.add(newBucket); } else { diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java index 6989a94c16fb9..a64d52b1ed7ce 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java @@ -26,6 +26,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.aggregations.AggregationsPlugin; +import org.elasticsearch.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.network.InetAddresses; @@ -56,7 +57,6 @@ import org.elasticsearch.search.aggregations.metrics.InternalStats; import org.elasticsearch.search.aggregations.metrics.Max; import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/CumulativeSumAggregatorTests.java similarity index 97% rename from server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java rename to modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/CumulativeSumAggregatorTests.java index 719680f4d508b..ecdd91fd73c35 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/CumulativeSumAggregatorTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.pipeline; +package org.elasticsearch.aggregations.pipeline; import org.apache.lucene.document.Document; import org.apache.lucene.document.NumericDocValuesField; @@ -19,11 +19,13 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.aggregations.AggregationsPlugin; import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.InternalAggregation; @@ -35,6 +37,8 @@ import org.elasticsearch.search.aggregations.metrics.InternalAvg; import org.elasticsearch.search.aggregations.metrics.Sum; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.CumulativeSumPipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import java.io.IOException; @@ -65,6 +69,11 @@ public class CumulativeSumAggregatorTests extends AggregatorTestCase { private static final List datasetValues = Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10); + @Override + protected List getSearchPlugins() { + return List.of(new AggregationsPlugin()); + } + public void testSimple() throws IOException { Query query = new MatchAllDocsQuery(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeAggregatorTests.java similarity index 98% rename from server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java rename to modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeAggregatorTests.java index 0369debdc47fb..3cf7dc917112e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeAggregatorTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.pipeline; +package org.elasticsearch.aggregations.pipeline; import org.apache.lucene.document.Document; import org.apache.lucene.document.NumericDocValuesField; @@ -19,11 +19,13 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.aggregations.AggregationsPlugin; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.InternalAggregation; @@ -36,7 +38,9 @@ import org.elasticsearch.search.aggregations.metrics.StatsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.Sum; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.BucketHelpers; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; +import org.elasticsearch.search.aggregations.pipeline.SimpleValue; import org.elasticsearch.search.aggregations.support.AggregationPath; import java.io.IOException; @@ -49,7 +53,6 @@ import static org.hamcrest.number.IsCloseTo.closeTo; public class DerivativeAggregatorTests extends AggregatorTestCase { - private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; private static int interval = 5; private static int numValueBuckets; @@ -69,6 +72,11 @@ public class DerivativeAggregatorTests extends AggregatorTestCase { private static Double[] firstDerivValueCounts_empty_rnd; private static long numDocsEmptyIdx_rnd; + @Override + protected List getSearchPlugins() { + return List.of(new AggregationsPlugin()); + } + private void setupValueCounts() { numDocsEmptyIdx = 0L; numDocsEmptyIdx_rnd = 0L; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativePipelinesAggregationBuilderTests.java similarity index 89% rename from modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeTests.java rename to modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativePipelinesAggregationBuilderTests.java index dcd958b604ea1..81c65ed1fceac 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativePipelinesAggregationBuilderTests.java @@ -8,21 +8,27 @@ package org.elasticsearch.aggregations.pipeline; +import org.elasticsearch.aggregations.AggregationsPlugin; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase; import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; -import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; import java.io.IOException; import java.util.HashSet; +import java.util.List; import java.util.Set; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; -public class DerivativeTests extends BasePipelineAggregationTestCase { +public class DerivativePipelinesAggregationBuilderTests extends BasePipelineAggregationTestCase { + @Override + protected List plugins() { + return List.of(new AggregationsPlugin()); + } @Override protected DerivativePipelineAggregationBuilder createTestAggregatorFactory() { @@ -75,4 +81,5 @@ public void testValidateException() throws IOException { ) ); } + } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalDerivativeTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeResultTests.java similarity index 65% rename from server/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalDerivativeTests.java rename to modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeResultTests.java index cd6e4a9387a95..8180e6482bd0d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/InternalDerivativeTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeResultTests.java @@ -6,27 +6,52 @@ * Side Public License, v 1. */ -package org.elasticsearch.search.aggregations.pipeline; +package org.elasticsearch.aggregations.pipeline; +import org.elasticsearch.aggregations.AggregationsPlugin; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.ParsedAggregation; +import org.elasticsearch.search.aggregations.pipeline.ParsedDerivative; import org.elasticsearch.test.InternalAggregationTestCase; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Stream; -public class InternalDerivativeTests extends InternalAggregationTestCase { +public class DerivativeResultTests extends InternalAggregationTestCase { + @Override + protected SearchPlugin registerPlugin() { + return new AggregationsPlugin(); + } + + @Override + protected List getNamedXContents() { + return Stream.concat( + super.getNamedXContents().stream(), + Stream.of( + new NamedXContentRegistry.Entry( + Aggregation.class, + new ParseField(DerivativePipelineAggregationBuilder.NAME), + (p, c) -> ParsedDerivative.fromXContent(p, (String) c) + ) + ) + ).toList(); + } @Override - protected InternalDerivative createTestInstance(String name, Map metadata) { + protected Derivative createTestInstance(String name, Map metadata) { DocValueFormat formatter = randomNumericDocValueFormat(); double value = frequently() ? randomDoubleBetween(-100000, 100000, true) : randomFrom(new Double[] { Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, Double.NaN }); double normalizationFactor = frequently() ? randomDoubleBetween(0, 100000, true) : 0; - return new InternalDerivative(name, value, normalizationFactor, formatter, metadata); + return new Derivative(name, value, normalizationFactor, formatter, metadata); } @Override @@ -35,12 +60,12 @@ public void testReduceRandom() { } @Override - protected void assertReduced(InternalDerivative reduced, List inputs) { + protected void assertReduced(Derivative reduced, List inputs) { // no test since reduce operation is unsupported } @Override - protected void assertFromXContent(InternalDerivative derivative, ParsedAggregation parsedAggregation) { + protected void assertFromXContent(Derivative derivative, ParsedAggregation parsedAggregation) { ParsedDerivative parsed = ((ParsedDerivative) parsedAggregation); if (Double.isInfinite(derivative.getValue()) == false && Double.isNaN(derivative.getValue()) == false) { assertEquals(derivative.getValue(), parsed.value(), Double.MIN_VALUE); @@ -53,7 +78,7 @@ protected void assertFromXContent(InternalDerivative derivative, ParsedAggregati } @Override - protected InternalDerivative mutateInstance(InternalDerivative instance) { + protected Derivative mutateInstance(Derivative instance) { String name = instance.getName(); double value = instance.getValue(); double normalizationFactor = instance.getNormalizationFactor(); @@ -79,6 +104,6 @@ protected InternalDerivative mutateInstance(InternalDerivative instance) { } default -> throw new AssertionError("Illegal randomisation branch"); } - return new InternalDerivative(name, value, normalizationFactor, formatter, metadata); + return new Derivative(name, value, normalizationFactor, formatter, metadata); } } diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/derivative.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/derivative.yml new file mode 100644 index 0000000000000..dd44101ea31f7 --- /dev/null +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/derivative.yml @@ -0,0 +1,372 @@ +setup: + - do: + bulk: + index: no_gaps + refresh: true + body: + - { "index": { } } + - { "n": 10, "@timestamp": "2022-01-01T00:00:00", "v": 1 } + - { "index": { } } + - { "n": 20, "@timestamp": "2022-01-01T01:00:00", "v": 2 } + - { "index": { } } + - { "n": 30, "@timestamp": "2022-01-01T02:00:00", "v": 1 } + - { "index": { } } + - { "n": 40, "@timestamp": "2022-01-01T03:00:00", "v": 4 } + - { "index": { } } + - { "n": 50, "@timestamp": "2022-01-01T04:00:00", "v": 5 } + - { "index": { } } + - { "n": 60, "@timestamp": "2022-01-01T05:00:00", "v": 9 } + + - do: + bulk: + index: gaps + refresh: true + body: + - { "index": { } } + - { "@timestamp": "2022-01-01T00:00:00", "v": 1 } + - { "index": { } } + - { "@timestamp": "2022-01-01T01:00:00", "v": 2 } + - { "index": { } } + - { "@timestamp": "2022-01-01T02:00:00", "v": 1 } + - { "index": { } } + - { "@timestamp": "2022-01-01T04:00:00", "v": 5 } + - { "index": { } } + - { "@timestamp": "2022-01-01T05:00:00", "v": 9 } + +--- +in date_histogram: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + derivative: + buckets_path: "v" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: -1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 3.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + +--- +in histogram: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + n: + histogram: + field: n + interval: 10 + aggs: + v: { avg: { field: v } } + d: + derivative: + buckets_path: "v" + - match: { hits.total.value: 6 } + - length: { aggregations.n.buckets: 6 } + - is_false: aggregations.n.buckets.0.d.value + - close_to: { aggregations.n.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.n.buckets.2.d.value: { value: -1.000, error: 0.0005 } } + - close_to: { aggregations.n.buckets.3.d.value: { value: 3.000, error: 0.0005 } } + - close_to: { aggregations.n.buckets.4.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.n.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + +--- +partially mapped: + - skip: + features: close_to + + - do: + search: + index: gaps,no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + n: { avg: { field: n } } + d: + derivative: + buckets_path: "n" + - match: { hits.total.value: 11 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 10.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 10.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 10.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 10.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 10.000, error: 0.0005 } } + +--- +format: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + derivative: + buckets_path: "v" + format: "0.00" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: -1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 3.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + - is_false: aggregations.@timestamp.buckets.0.d.value_as_string + - match: { aggregations.@timestamp.buckets.1.d.value_as_string: "1.00" } + - match: { aggregations.@timestamp.buckets.2.d.value_as_string: "-1.00" } + - match: { aggregations.@timestamp.buckets.3.d.value_as_string: "3.00" } + - match: { aggregations.@timestamp.buckets.4.d.value_as_string: "1.00" } + - match: { aggregations.@timestamp.buckets.5.d.value_as_string: "4.00" } + +--- +gap_policy=skip: + - skip: + features: close_to + + - do: + search: + index: gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + derivative: + buckets_path: "v" + gap_policy: skip + - match: { hits.total.value: 5 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: -1.000, error: 0.0005 } } + - is_false: aggregations.@timestamp.buckets.3.d.value + - is_false: aggregations.@timestamp.buckets.4.d.value + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + +--- +gap_policy=insert_zeros: + - skip: + features: close_to + + - do: + search: + index: gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + derivative: + buckets_path: "v" + gap_policy: insert_zeros + - match: { hits.total.value: 5 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: -1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: -1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 5.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + +--- +gap_policy=keep_values: + - skip: + features: close_to + + - do: + search: + index: gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + derivative: + buckets_path: "v" + gap_policy: keep_values + - match: { hits.total.value: 5 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: -1.000, error: 0.0005 } } + - is_false: aggregations.@timestamp.buckets.3.d.value + - is_false: aggregations.@timestamp.buckets.4.d.value + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + +--- +dotted name: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + "v.v": { avg: { field: v } } + d: + derivative: + buckets_path: "v.v.value" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: -1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 3.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + +--- +dotted value: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: + percentiles: + field: v + percents: [ 50, 99.9 ] + d: + derivative: + buckets_path: "v[99.9]" + - match: { hits.total.value: 6 } + - length: { aggregations.@timestamp.buckets: 6 } + - is_false: aggregations.@timestamp.buckets.0.d.value + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: -1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 3.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } + +--- +no results: + - skip: + features: close_to + + - do: + search: + index: no_gaps + body: + size: 0 + query: + match: + missing_field: not found + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + derivative: + buckets_path: "v" + - match: { hits.total.value: 0 } + - length: { aggregations.@timestamp.buckets: 0 } + +--- +bad path: + - do: + catch: '/Validation Failed: 1: No aggregation found for path \[missing\];/' + search: + index: no_gaps + body: + size: 0 + query: + match: + missing_field: not found + aggs: + "@timestamp": + date_histogram: + field: "@timestamp" + fixed_interval: 1h + aggs: + v: { avg: { field: v } } + d: + derivative: + buckets_path: "missing" + +--- +Not under date_histo: + - do: + catch: /derivative aggregation \[d\] must have a histogram, date_histogram or auto_date_histogram as parent but doesn't have a parent/ + search: + rest_total_hits_as_int: true + body: + size: 0 + aggs: + the_avg: + avg: + field: "value_field" + d: + derivative: + buckets_path: "the_avg" diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java index 98f385a98a182..0dfbf854bc002 100644 --- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java +++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java @@ -64,7 +64,7 @@ public void setUp() throws Exception { protected List getNamedXContents() { ContextParser parser = (p, c) -> ParsedMatrixStats.fromXContent(p, (String) c); return CollectionUtils.appendToCopy( - getDefaultNamedXContents(), + super.getNamedXContents(), new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(MatrixStatsAggregationBuilder.NAME), parser) ); } diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java index 034c5d48e325f..a21917a41e308 100644 --- a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java +++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; @@ -54,7 +55,6 @@ import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.builder.SearchSourceBuilder; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java deleted file mode 100644 index 6750946d27eb5..0000000000000 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java +++ /dev/null @@ -1,670 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; -import org.elasticsearch.search.aggregations.metrics.Stats; -import org.elasticsearch.search.aggregations.metrics.Sum; -import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; -import org.elasticsearch.search.aggregations.support.AggregationPath; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.hamcrest.Matchers; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; -import static org.elasticsearch.search.aggregations.AggregationBuilders.filters; -import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; -import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; -import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; -import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.derivative; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.core.IsNull.notNullValue; -import static org.hamcrest.core.IsNull.nullValue; - -@ESIntegTestCase.SuiteScopeTestCase -public class DerivativeIT extends ESIntegTestCase { - - private static final String SINGLE_VALUED_FIELD_NAME = "l_value"; - - private static int interval; - private static int numValueBuckets; - private static int numFirstDerivValueBuckets; - private static int numSecondDerivValueBuckets; - private static long[] valueCounts; - private static long[] firstDerivValueCounts; - private static long[] secondDerivValueCounts; - - private static Long[] valueCounts_empty; - private static long numDocsEmptyIdx; - private static Double[] firstDerivValueCounts_empty; - - // expected bucket values for random setup with gaps - private static int numBuckets_empty_rnd; - private static Long[] valueCounts_empty_rnd; - private static Double[] firstDerivValueCounts_empty_rnd; - private static long numDocsEmptyIdx_rnd; - - @Override - public void setupSuiteScopeCluster() throws Exception { - createIndex("idx"); - createIndex("idx_unmapped"); - - interval = 5; - numValueBuckets = randomIntBetween(6, 80); - - valueCounts = new long[numValueBuckets]; - for (int i = 0; i < numValueBuckets; i++) { - valueCounts[i] = randomIntBetween(1, 20); - } - - numFirstDerivValueBuckets = numValueBuckets - 1; - firstDerivValueCounts = new long[numFirstDerivValueBuckets]; - Long lastValueCount = null; - for (int i = 0; i < numValueBuckets; i++) { - long thisValue = valueCounts[i]; - if (lastValueCount != null) { - long diff = thisValue - lastValueCount; - firstDerivValueCounts[i - 1] = diff; - } - lastValueCount = thisValue; - } - - numSecondDerivValueBuckets = numFirstDerivValueBuckets - 1; - secondDerivValueCounts = new long[numSecondDerivValueBuckets]; - Long lastFirstDerivativeValueCount = null; - for (int i = 0; i < numFirstDerivValueBuckets; i++) { - long thisFirstDerivativeValue = firstDerivValueCounts[i]; - if (lastFirstDerivativeValueCount != null) { - long diff = thisFirstDerivativeValue - lastFirstDerivativeValueCount; - secondDerivValueCounts[i - 1] = diff; - } - lastFirstDerivativeValueCount = thisFirstDerivativeValue; - } - - List builders = new ArrayList<>(); - for (int i = 0; i < numValueBuckets; i++) { - for (int docs = 0; docs < valueCounts[i]; docs++) { - builders.add(client().prepareIndex("idx").setSource(newDocBuilder(i * interval))); - } - } - - // setup for index with empty buckets - valueCounts_empty = new Long[] { 1L, 1L, 2L, 0L, 2L, 2L, 0L, 0L, 0L, 3L, 2L, 1L }; - firstDerivValueCounts_empty = new Double[] { null, 0d, 1d, -2d, 2d, 0d, -2d, 0d, 0d, 3d, -1d, -1d }; - - assertAcked(prepareCreate("empty_bucket_idx").setMapping(SINGLE_VALUED_FIELD_NAME, "type=integer")); - for (int i = 0; i < valueCounts_empty.length; i++) { - for (int docs = 0; docs < valueCounts_empty[i]; docs++) { - builders.add(client().prepareIndex("empty_bucket_idx").setSource(newDocBuilder(i))); - numDocsEmptyIdx++; - } - } - - // randomized setup for index with empty buckets - numBuckets_empty_rnd = randomIntBetween(20, 100); - valueCounts_empty_rnd = new Long[numBuckets_empty_rnd]; - firstDerivValueCounts_empty_rnd = new Double[numBuckets_empty_rnd]; - firstDerivValueCounts_empty_rnd[0] = null; - - assertAcked(prepareCreate("empty_bucket_idx_rnd").setMapping(SINGLE_VALUED_FIELD_NAME, "type=integer")); - for (int i = 0; i < numBuckets_empty_rnd; i++) { - valueCounts_empty_rnd[i] = (long) randomIntBetween(1, 10); - // make approximately half of the buckets empty - if (randomBoolean()) valueCounts_empty_rnd[i] = 0L; - for (int docs = 0; docs < valueCounts_empty_rnd[i]; docs++) { - builders.add(client().prepareIndex("empty_bucket_idx_rnd").setSource(newDocBuilder(i))); - numDocsEmptyIdx_rnd++; - } - if (i > 0) { - firstDerivValueCounts_empty_rnd[i] = (double) valueCounts_empty_rnd[i] - valueCounts_empty_rnd[i - 1]; - } - } - - indexRandom(true, builders); - ensureSearchable(); - } - - private XContentBuilder newDocBuilder(int singleValueFieldValue) throws IOException { - return jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, singleValueFieldValue).endObject(); - } - - /** - * test first and second derivative on the sing - */ - public void testDocCountDerivative() { - - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME) - .interval(interval) - .subAggregation(derivative("deriv", "_count")) - .subAggregation(derivative("2nd_deriv", "deriv")) - ) - .get(); - - assertSearchResponse(response); - - Histogram deriv = response.getAggregations().get("histo"); - assertThat(deriv, notNullValue()); - assertThat(deriv.getName(), equalTo("histo")); - List buckets = deriv.getBuckets(); - assertThat(buckets.size(), equalTo(numValueBuckets)); - - for (int i = 0; i < numValueBuckets; ++i) { - Histogram.Bucket bucket = buckets.get(i); - checkBucketKeyAndDocCount("InternalBucket " + i, bucket, i * interval, valueCounts[i]); - SimpleValue docCountDeriv = bucket.getAggregations().get("deriv"); - if (i > 0) { - assertThat(docCountDeriv, notNullValue()); - assertThat(docCountDeriv.value(), equalTo((double) firstDerivValueCounts[i - 1])); - } else { - assertThat(docCountDeriv, nullValue()); - } - SimpleValue docCount2ndDeriv = bucket.getAggregations().get("2nd_deriv"); - if (i > 1) { - assertThat(docCount2ndDeriv, notNullValue()); - assertThat(docCount2ndDeriv.value(), equalTo((double) secondDerivValueCounts[i - 2])); - } else { - assertThat(docCount2ndDeriv, nullValue()); - } - } - } - - /** - * test first and second derivative on the sing - */ - public void testSingleValuedField_normalised() { - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME) - .interval(interval) - .minDocCount(0) - .subAggregation(derivative("deriv", "_count").unit("1ms")) - .subAggregation(derivative("2nd_deriv", "deriv").unit("10ms")) - ) - .get(); - - assertSearchResponse(response); - - Histogram deriv = response.getAggregations().get("histo"); - assertThat(deriv, notNullValue()); - assertThat(deriv.getName(), equalTo("histo")); - List buckets = deriv.getBuckets(); - assertThat(buckets.size(), equalTo(numValueBuckets)); - - for (int i = 0; i < numValueBuckets; ++i) { - Histogram.Bucket bucket = buckets.get(i); - checkBucketKeyAndDocCount("InternalBucket " + i, bucket, i * interval, valueCounts[i]); - Derivative docCountDeriv = bucket.getAggregations().get("deriv"); - if (i > 0) { - assertThat(docCountDeriv, notNullValue()); - assertThat(docCountDeriv.value(), closeTo((firstDerivValueCounts[i - 1]), 0.00001)); - assertThat(docCountDeriv.normalizedValue(), closeTo((double) (firstDerivValueCounts[i - 1]) / 5, 0.00001)); - } else { - assertThat(docCountDeriv, nullValue()); - } - Derivative docCount2ndDeriv = bucket.getAggregations().get("2nd_deriv"); - if (i > 1) { - assertThat(docCount2ndDeriv, notNullValue()); - assertThat(docCount2ndDeriv.value(), closeTo((secondDerivValueCounts[i - 2]), 0.00001)); - assertThat(docCount2ndDeriv.normalizedValue(), closeTo((double) (secondDerivValueCounts[i - 2]) * 2, 0.00001)); - } else { - assertThat(docCount2ndDeriv, nullValue()); - } - } - } - - public void testSingleValueAggDerivative() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME) - .interval(interval) - .subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME)) - .subAggregation(derivative("deriv", "sum")) - ) - .get(); - - assertSearchResponse(response); - - Histogram deriv = response.getAggregations().get("histo"); - assertThat(deriv, notNullValue()); - assertThat(deriv.getName(), equalTo("histo")); - assertThat(deriv.getBuckets().size(), equalTo(numValueBuckets)); - Object[] propertiesKeys = (Object[]) ((InternalAggregation) deriv).getProperty("_key"); - Object[] propertiesDocCounts = (Object[]) ((InternalAggregation) deriv).getProperty("_count"); - Object[] propertiesSumCounts = (Object[]) ((InternalAggregation) deriv).getProperty("sum.value"); - - List buckets = new ArrayList<>(deriv.getBuckets()); - Long expectedSumPreviousBucket = Long.MIN_VALUE; // start value, gets - // overwritten - for (int i = 0; i < numValueBuckets; ++i) { - Histogram.Bucket bucket = buckets.get(i); - checkBucketKeyAndDocCount("InternalBucket " + i, bucket, i * interval, valueCounts[i]); - Sum sum = bucket.getAggregations().get("sum"); - assertThat(sum, notNullValue()); - long expectedSum = valueCounts[i] * (i * interval); - assertThat(sum.value(), equalTo((double) expectedSum)); - SimpleValue sumDeriv = bucket.getAggregations().get("deriv"); - if (i > 0) { - assertThat(sumDeriv, notNullValue()); - long sumDerivValue = expectedSum - expectedSumPreviousBucket; - assertThat(sumDeriv.value(), equalTo((double) sumDerivValue)); - assertThat( - ((InternalMultiBucketAggregation.InternalBucket) bucket).getProperty( - "histo", - AggregationPath.parse("deriv.value").getPathElementsAsStringList() - ), - equalTo((double) sumDerivValue) - ); - } else { - assertThat(sumDeriv, nullValue()); - } - expectedSumPreviousBucket = expectedSum; - assertThat(propertiesKeys[i], equalTo((double) i * interval)); - assertThat((long) propertiesDocCounts[i], equalTo(valueCounts[i])); - assertThat((double) propertiesSumCounts[i], equalTo((double) expectedSum)); - } - } - - public void testMultiValueAggDerivative() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME) - .interval(interval) - .subAggregation(stats("stats").field(SINGLE_VALUED_FIELD_NAME)) - .subAggregation(derivative("deriv", "stats.sum")) - ) - .get(); - - assertSearchResponse(response); - - Histogram deriv = response.getAggregations().get("histo"); - assertThat(deriv, notNullValue()); - assertThat(deriv.getName(), equalTo("histo")); - assertThat(deriv.getBuckets().size(), equalTo(numValueBuckets)); - Object[] propertiesKeys = (Object[]) ((InternalAggregation) deriv).getProperty("_key"); - Object[] propertiesDocCounts = (Object[]) ((InternalAggregation) deriv).getProperty("_count"); - Object[] propertiesSumCounts = (Object[]) ((InternalAggregation) deriv).getProperty("stats.sum"); - - List buckets = new ArrayList<>(deriv.getBuckets()); - Long expectedSumPreviousBucket = Long.MIN_VALUE; // start value, gets - // overwritten - for (int i = 0; i < numValueBuckets; ++i) { - Histogram.Bucket bucket = buckets.get(i); - checkBucketKeyAndDocCount("InternalBucket " + i, bucket, i * interval, valueCounts[i]); - Stats stats = bucket.getAggregations().get("stats"); - assertThat(stats, notNullValue()); - long expectedSum = valueCounts[i] * (i * interval); - assertThat(stats.getSum(), equalTo((double) expectedSum)); - SimpleValue sumDeriv = bucket.getAggregations().get("deriv"); - if (i > 0) { - assertThat(sumDeriv, notNullValue()); - long sumDerivValue = expectedSum - expectedSumPreviousBucket; - assertThat(sumDeriv.value(), equalTo((double) sumDerivValue)); - assertThat( - ((InternalMultiBucketAggregation.InternalBucket) bucket).getProperty( - "histo", - AggregationPath.parse("deriv.value").getPathElementsAsStringList() - ), - equalTo((double) sumDerivValue) - ); - } else { - assertThat(sumDeriv, nullValue()); - } - expectedSumPreviousBucket = expectedSum; - assertThat(propertiesKeys[i], equalTo((double) i * interval)); - assertThat((long) propertiesDocCounts[i], equalTo(valueCounts[i])); - assertThat((double) propertiesSumCounts[i], equalTo((double) expectedSum)); - } - } - - public void testUnmapped() throws Exception { - SearchResponse response = client().prepareSearch("idx_unmapped") - .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).subAggregation(derivative("deriv", "_count")) - ) - .get(); - - assertSearchResponse(response); - - Histogram deriv = response.getAggregations().get("histo"); - assertThat(deriv, notNullValue()); - assertThat(deriv.getName(), equalTo("histo")); - assertThat(deriv.getBuckets().size(), equalTo(0)); - } - - public void testPartiallyUnmapped() throws Exception { - SearchResponse response = client().prepareSearch("idx", "idx_unmapped") - .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).subAggregation(derivative("deriv", "_count")) - ) - .get(); - - assertSearchResponse(response); - - Histogram deriv = response.getAggregations().get("histo"); - assertThat(deriv, notNullValue()); - assertThat(deriv.getName(), equalTo("histo")); - List buckets = deriv.getBuckets(); - assertThat(deriv.getBuckets().size(), equalTo(numValueBuckets)); - - for (int i = 0; i < numValueBuckets; ++i) { - Histogram.Bucket bucket = buckets.get(i); - checkBucketKeyAndDocCount("InternalBucket " + i, bucket, i * interval, valueCounts[i]); - SimpleValue docCountDeriv = bucket.getAggregations().get("deriv"); - if (i > 0) { - assertThat(docCountDeriv, notNullValue()); - assertThat(docCountDeriv.value(), equalTo((double) firstDerivValueCounts[i - 1])); - } else { - assertThat(docCountDeriv, nullValue()); - } - } - } - - public void testDocCountDerivativeWithGaps() throws Exception { - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") - .setQuery(matchAllQuery()) - .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1).subAggregation(derivative("deriv", "_count"))) - .get(); - - assertThat(searchResponse.getHits().getTotalHits().value, equalTo(numDocsEmptyIdx)); - - Histogram deriv = searchResponse.getAggregations().get("histo"); - assertThat(deriv, Matchers.notNullValue()); - assertThat(deriv.getName(), equalTo("histo")); - List buckets = deriv.getBuckets(); - assertThat(buckets.size(), equalTo(valueCounts_empty.length)); - - for (int i = 0; i < valueCounts_empty.length; i++) { - Histogram.Bucket bucket = buckets.get(i); - checkBucketKeyAndDocCount("InternalBucket " + i, bucket, i, valueCounts_empty[i]); - SimpleValue docCountDeriv = bucket.getAggregations().get("deriv"); - if (firstDerivValueCounts_empty[i] == null) { - assertThat(docCountDeriv, nullValue()); - } else { - assertThat(docCountDeriv.value(), equalTo(firstDerivValueCounts_empty[i])); - } - } - } - - public void testDocCountDerivativeWithGaps_random() throws Exception { - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx_rnd") - .setQuery(matchAllQuery()) - .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME) - .interval(1) - .extendedBounds(0L, numBuckets_empty_rnd - 1) - .subAggregation(derivative("deriv", "_count").gapPolicy(randomFrom(GapPolicy.values()))) - ) - .get(); - - assertThat(searchResponse.getHits().getTotalHits().value, equalTo(numDocsEmptyIdx_rnd)); - - Histogram deriv = searchResponse.getAggregations().get("histo"); - assertThat(deriv, Matchers.notNullValue()); - assertThat(deriv.getName(), equalTo("histo")); - List buckets = deriv.getBuckets(); - assertThat(buckets.size(), equalTo(numBuckets_empty_rnd)); - - for (int i = 0; i < valueCounts_empty_rnd.length; i++) { - Histogram.Bucket bucket = buckets.get(i); - checkBucketKeyAndDocCount("InternalBucket " + i, bucket, i, valueCounts_empty_rnd[i]); - SimpleValue docCountDeriv = bucket.getAggregations().get("deriv"); - if (firstDerivValueCounts_empty_rnd[i] == null) { - assertThat(docCountDeriv, nullValue()); - } else { - assertThat(docCountDeriv.value(), equalTo(firstDerivValueCounts_empty_rnd[i])); - } - } - } - - public void testDocCountDerivativeWithGaps_insertZeros() throws Exception { - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") - .setQuery(matchAllQuery()) - .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME) - .interval(1) - .subAggregation(derivative("deriv", "_count").gapPolicy(GapPolicy.INSERT_ZEROS)) - ) - .get(); - - assertThat(searchResponse.getHits().getTotalHits().value, equalTo(numDocsEmptyIdx)); - - Histogram deriv = searchResponse.getAggregations().get("histo"); - assertThat(deriv, Matchers.notNullValue()); - assertThat(deriv.getName(), equalTo("histo")); - List buckets = deriv.getBuckets(); - assertThat(buckets.size(), equalTo(valueCounts_empty.length)); - - for (int i = 0; i < valueCounts_empty.length; i++) { - Histogram.Bucket bucket = buckets.get(i); - checkBucketKeyAndDocCount("InternalBucket " + i + ": ", bucket, i, valueCounts_empty[i]); - SimpleValue docCountDeriv = bucket.getAggregations().get("deriv"); - if (firstDerivValueCounts_empty[i] == null) { - assertThat(docCountDeriv, nullValue()); - } else { - assertThat(docCountDeriv.value(), equalTo(firstDerivValueCounts_empty[i])); - } - } - } - - public void testSingleValueAggDerivativeWithGaps() throws Exception { - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") - .setQuery(matchAllQuery()) - .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME) - .interval(1) - .subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME)) - .subAggregation(derivative("deriv", "sum")) - ) - .get(); - - assertThat(searchResponse.getHits().getTotalHits().value, equalTo(numDocsEmptyIdx)); - - Histogram deriv = searchResponse.getAggregations().get("histo"); - assertThat(deriv, Matchers.notNullValue()); - assertThat(deriv.getName(), equalTo("histo")); - List buckets = deriv.getBuckets(); - assertThat(buckets.size(), equalTo(valueCounts_empty.length)); - - double lastSumValue = Double.NaN; - for (int i = 0; i < valueCounts_empty.length; i++) { - Histogram.Bucket bucket = buckets.get(i); - checkBucketKeyAndDocCount("InternalBucket " + i, bucket, i, valueCounts_empty[i]); - Sum sum = bucket.getAggregations().get("sum"); - double thisSumValue = sum.value(); - if (bucket.getDocCount() == 0) { - thisSumValue = Double.NaN; - } - SimpleValue sumDeriv = bucket.getAggregations().get("deriv"); - if (i == 0) { - assertThat(sumDeriv, nullValue()); - } else { - double expectedDerivative = thisSumValue - lastSumValue; - if (Double.isNaN(expectedDerivative)) { - assertThat(sumDeriv.value(), equalTo(expectedDerivative)); - } else { - assertThat(sumDeriv.value(), closeTo(expectedDerivative, 0.00001)); - } - } - lastSumValue = thisSumValue; - } - } - - public void testSingleValueAggDerivativeWithGaps_insertZeros() throws Exception { - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx") - .setQuery(matchAllQuery()) - .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME) - .interval(1) - .subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME)) - .subAggregation(derivative("deriv", "sum").gapPolicy(GapPolicy.INSERT_ZEROS)) - ) - .get(); - - assertThat(searchResponse.getHits().getTotalHits().value, equalTo(numDocsEmptyIdx)); - - Histogram deriv = searchResponse.getAggregations().get("histo"); - assertThat(deriv, Matchers.notNullValue()); - assertThat(deriv.getName(), equalTo("histo")); - List buckets = deriv.getBuckets(); - assertThat(buckets.size(), equalTo(valueCounts_empty.length)); - - double lastSumValue = Double.NaN; - for (int i = 0; i < valueCounts_empty.length; i++) { - Histogram.Bucket bucket = buckets.get(i); - checkBucketKeyAndDocCount("InternalBucket " + i, bucket, i, valueCounts_empty[i]); - Sum sum = bucket.getAggregations().get("sum"); - double thisSumValue = sum.value(); - if (bucket.getDocCount() == 0) { - thisSumValue = 0; - } - SimpleValue sumDeriv = bucket.getAggregations().get("deriv"); - if (i == 0) { - assertThat(sumDeriv, nullValue()); - } else { - double expectedDerivative = thisSumValue - lastSumValue; - assertThat(sumDeriv.value(), closeTo(expectedDerivative, 0.00001)); - } - lastSumValue = thisSumValue; - } - } - - public void testSingleValueAggDerivativeWithGaps_random() throws Exception { - GapPolicy gapPolicy = randomFrom(GapPolicy.values()); - SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx_rnd") - .setQuery(matchAllQuery()) - .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME) - .interval(1) - .extendedBounds(0L, (long) numBuckets_empty_rnd - 1) - .subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME)) - .subAggregation(derivative("deriv", "sum").gapPolicy(gapPolicy)) - ) - .get(); - - assertThat(searchResponse.getHits().getTotalHits().value, equalTo(numDocsEmptyIdx_rnd)); - - Histogram deriv = searchResponse.getAggregations().get("histo"); - assertThat(deriv, Matchers.notNullValue()); - assertThat(deriv.getName(), equalTo("histo")); - List buckets = deriv.getBuckets(); - assertThat(buckets.size(), equalTo(numBuckets_empty_rnd)); - - double lastSumValue = Double.NaN; - for (int i = 0; i < valueCounts_empty_rnd.length; i++) { - Histogram.Bucket bucket = buckets.get(i); - checkBucketKeyAndDocCount("InternalBucket " + i, bucket, i, valueCounts_empty_rnd[i]); - Sum sum = bucket.getAggregations().get("sum"); - double thisSumValue = sum.value(); - if (bucket.getDocCount() == 0 && gapPolicy != GapPolicy.KEEP_VALUES) { - thisSumValue = gapPolicy == GapPolicy.INSERT_ZEROS ? 0 : Double.NaN; - } - SimpleValue sumDeriv = bucket.getAggregations().get("deriv"); - if (i == 0) { - assertThat(sumDeriv, nullValue()); - } else { - double expectedDerivative = thisSumValue - lastSumValue; - if (Double.isNaN(expectedDerivative)) { - assertThat(sumDeriv.value(), equalTo(expectedDerivative)); - } else { - assertThat(sumDeriv.value(), closeTo(expectedDerivative, 0.00001)); - } - } - lastSumValue = thisSumValue; - } - } - - public void testSingleValueAggDerivative_invalidPath() throws Exception { - try { - client().prepareSearch("idx") - .addAggregation( - histogram("histo").field(SINGLE_VALUED_FIELD_NAME) - .interval(interval) - .subAggregation( - filters("filters", QueryBuilders.termQuery("tag", "foo")).subAggregation( - sum("sum").field(SINGLE_VALUED_FIELD_NAME) - ) - ) - .subAggregation(derivative("deriv", "filters>get>sum")) - ) - .get(); - fail("Expected an Exception but didn't get one"); - } catch (Exception e) { - Throwable cause = ExceptionsHelper.unwrapCause(e); - if (cause == null) { - throw e; - } else if (cause instanceof SearchPhaseExecutionException) { - SearchPhaseExecutionException spee = (SearchPhaseExecutionException) e; - Throwable rootCause = spee.getRootCause(); - if ((rootCause instanceof IllegalArgumentException) == false) { - throw e; - } - } else if ((cause instanceof IllegalArgumentException) == false) { - throw e; - } - } - } - - public void testDerivDerivNPE() throws Exception { - createIndex("deriv_npe"); - - for (int i = 0; i < 10; i++) { - Integer value = i; - if (i == 1 || i == 3) { - value = null; - } - - XContentBuilder doc = jsonBuilder().startObject().field("tick", i).field("value", value).endObject(); - client().prepareIndex("deriv_npe").setSource(doc).get(); - } - - refresh(); - - SearchResponse response = client().prepareSearch("deriv_npe") - .addAggregation( - histogram("histo").field("tick") - .interval(1) - .subAggregation(avg("avg").field("value")) - .subAggregation(derivative("deriv1", "avg")) - .subAggregation(derivative("deriv2", "deriv1")) - ) - .get(); - - assertSearchResponse(response); - } - - private void checkBucketKeyAndDocCount( - final String msg, - final Histogram.Bucket bucket, - final long expectedKey, - final long expectedDocCount - ) { - assertThat(msg, bucket, notNullValue()); - assertThat(msg + " key", ((Number) bucket.getKey()).longValue(), equalTo(expectedKey)); - assertThat(msg + " docCount", bucket.getDocCount(), equalTo(expectedDocCount)); - } -} diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 628ff8fbe1b52..fe29b9bf9f14e 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -192,11 +192,9 @@ import org.elasticsearch.search.aggregations.pipeline.BucketSelectorPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketSortPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.CumulativeSumPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketParser; import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.InternalBucketMetricValue; -import org.elasticsearch.search.aggregations.pipeline.InternalDerivative; import org.elasticsearch.search.aggregations.pipeline.InternalExtendedStatsBucket; import org.elasticsearch.search.aggregations.pipeline.InternalPercentilesBucket; import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; @@ -693,13 +691,6 @@ private void registerAggregation(AggregationSpec spec, ValuesSourceRegistry.Buil } private void registerPipelineAggregations(List plugins) { - registerPipelineAggregation( - new PipelineAggregationSpec( - DerivativePipelineAggregationBuilder.NAME, - DerivativePipelineAggregationBuilder::new, - DerivativePipelineAggregationBuilder::parse - ).addResultReader(InternalDerivative::new) - ); registerPipelineAggregation( new PipelineAggregationSpec( MaxBucketPipelineAggregationBuilder.NAME, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilders.java b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilders.java index e5988ced78516..3c5178d16117d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilders.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilders.java @@ -14,7 +14,6 @@ import org.elasticsearch.search.aggregations.pipeline.BucketSelectorPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketSortPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.CumulativeSumPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.MinBucketPipelineAggregationBuilder; @@ -31,10 +30,6 @@ public final class PipelineAggregatorBuilders { private PipelineAggregatorBuilders() {} - public static DerivativePipelineAggregationBuilder derivative(String name, String bucketsPath) { - return new DerivativePipelineAggregationBuilder(name, bucketsPath); - } - public static MaxBucketPipelineAggregationBuilder maxBucket(String name, String bucketsPath) { return new MaxBucketPipelineAggregationBuilder(name, bucketsPath); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricInspectionHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricInspectionHelper.java index 1fbab64a17a36..038888398c4c5 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricInspectionHelper.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MetricInspectionHelper.java @@ -7,8 +7,6 @@ */ package org.elasticsearch.search.aggregations.metrics; -import org.elasticsearch.search.aggregations.pipeline.InternalDerivative; - /** * Counterpart to {@link org.elasticsearch.search.aggregations.support.AggregationInspectionHelper}, providing * helpers for some aggs that have package-private getters. AggregationInspectionHelper delegates to these @@ -60,8 +58,4 @@ public static boolean hasValue(InternalTopHits agg) { public static boolean hasValue(InternalWeightedAvg agg) { return (agg.getSum() == 0.0 && agg.getWeight() == 0L) == false; } - - public static boolean hasValue(InternalDerivative agg) { - return true; - } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/Derivative.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/Derivative.java deleted file mode 100644 index 77259d343dd83..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/Derivative.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -public interface Derivative extends SimpleValue { - - /** - * Returns the normalized value. If no normalised factor has been specified - * this method will return {@link #value()} - * - * @return the normalized value - */ - double normalizedValue(); -} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedDerivative.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedDerivative.java index b227cebb3e58a..66d2c0621b410 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedDerivative.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedDerivative.java @@ -16,7 +16,7 @@ import java.io.IOException; -public class ParsedDerivative extends ParsedSimpleValue implements Derivative { +public class ParsedDerivative extends ParsedSimpleValue { private double normalizedValue; private String normalizedAsString; @@ -24,14 +24,19 @@ public class ParsedDerivative extends ParsedSimpleValue implements Derivative { private static final ParseField NORMALIZED_AS_STRING = new ParseField("normalized_value_as_string"); private static final ParseField NORMALIZED = new ParseField("normalized_value"); - @Override + /** + * Returns the normalized value. If no normalised factor has been specified + * this method will return {@link #value()} + * + * @return the normalized value + */ public double normalizedValue() { return this.normalizedValue; } @Override public String getType() { - return DerivativePipelineAggregationBuilder.NAME; + return "derivative"; } private static final ObjectParser PARSER = new ObjectParser<>( diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java index fa3bd155a001e..4c81141be2ead 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java @@ -34,8 +34,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.heuristic.ChiSquare; import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.InternalDerivative; import org.elasticsearch.search.aggregations.pipeline.MovAvgPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.AggregationContext; @@ -66,6 +64,7 @@ import org.elasticsearch.search.suggest.term.TermSuggestion; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; @@ -84,6 +83,7 @@ import static java.util.Collections.singletonMap; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -104,7 +104,7 @@ public Map getHighlighters() { SearchPlugin registersDupeSuggester = new SearchPlugin() { @Override public List> getSuggesters() { - return singletonList( + return List.of( new SuggesterSpec<>( TermSuggestionBuilder.SUGGESTION_NAME, TermSuggestionBuilder::new, @@ -119,7 +119,7 @@ public List> getSuggesters() { SearchPlugin registersDupeScoreFunction = new SearchPlugin() { @Override public List> getScoreFunctions() { - return singletonList( + return List.of( new ScoreFunctionSpec<>( GaussDecayFunctionBuilder.NAME, GaussDecayFunctionBuilder::new, @@ -133,7 +133,7 @@ public List> getScoreFunctions() { SearchPlugin registersDupeSignificanceHeuristic = new SearchPlugin() { @Override public List> getSignificanceHeuristics() { - return singletonList(new SignificanceHeuristicSpec<>(ChiSquare.NAME, ChiSquare::new, ChiSquare.PARSER)); + return List.of(new SignificanceHeuristicSpec<>(ChiSquare.NAME, ChiSquare::new, ChiSquare.PARSER)); } }; expectThrows(IllegalArgumentException.class, registryForPlugin(registersDupeSignificanceHeuristic)); @@ -141,7 +141,7 @@ public List> getSignificanceHeuristics() { SearchPlugin registersDupeFetchSubPhase = new SearchPlugin() { @Override public List getFetchSubPhases(FetchPhaseConstructionContext context) { - return singletonList(new ExplainPhase()); + return List.of(new ExplainPhase()); } }; expectThrows(IllegalArgumentException.class, registryForPlugin(registersDupeFetchSubPhase)); @@ -149,7 +149,7 @@ public List getFetchSubPhases(FetchPhaseConstructionContext conte SearchPlugin registersDupeQuery = new SearchPlugin() { @Override public List> getQueries() { - return singletonList(new QuerySpec<>(TermQueryBuilder.NAME, TermQueryBuilder::new, TermQueryBuilder::fromXContent)); + return List.of(new QuerySpec<>(TermQueryBuilder.NAME, TermQueryBuilder::new, TermQueryBuilder::fromXContent)); } }; expectThrows(IllegalArgumentException.class, registryForPlugin(registersDupeQuery)); @@ -157,7 +157,7 @@ public List> getQueries() { SearchPlugin registersDupeAggregation = new SearchPlugin() { @Override public List getAggregations() { - return singletonList( + return List.of( new AggregationSpec(TermsAggregationBuilder.NAME, TermsAggregationBuilder::new, TermsAggregationBuilder.PARSER) ); } @@ -167,12 +167,9 @@ public List getAggregations() { SearchPlugin registersDupePipelineAggregation = new SearchPlugin() { @Override public List getPipelineAggregations() { - return singletonList( - new PipelineAggregationSpec( - DerivativePipelineAggregationBuilder.NAME, - DerivativePipelineAggregationBuilder::new, - DerivativePipelineAggregationBuilder::parse - ).addResultReader(InternalDerivative::new) + return List.of( + new PipelineAggregationSpec("test", TestPipelineAggregationBuilder::new, TestPipelineAggregationBuilder.PARSER), + new PipelineAggregationSpec("test", TestPipelineAggregationBuilder::new, TestPipelineAggregationBuilder.PARSER) ); } }; @@ -181,7 +178,7 @@ public List getPipelineAggregations() { SearchPlugin registersDupeRescorer = new SearchPlugin() { @Override public List> getRescorers() { - return singletonList( + return List.of( new RescorerSpec<>(QueryRescorerBuilder.NAME, QueryRescorerBuilder::new, QueryRescorerBuilder::fromXContent) ); } @@ -355,7 +352,7 @@ public void testRegisterPipelineAggregation() { @Override public List getPipelineAggregations() { return singletonList( - new PipelineAggregationSpec("test", TestPipelineAggregationBuilder::new, TestPipelineAggregationBuilder::fromXContent) + new PipelineAggregationSpec("test", TestPipelineAggregationBuilder::new, TestPipelineAggregationBuilder.PARSER) ); } })); @@ -556,6 +553,19 @@ public Version getMinimalSupportedVersion() { * Dummy test {@link PipelineAggregator} used to test registering aggregation builders. */ private static class TestPipelineAggregationBuilder extends AbstractPipelineAggregationBuilder { + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "test", + false, + (args, name) -> { return new TestPipelineAggregationBuilder(name, (String) args[0]); } + ); + static { + PARSER.declareString(constructorArg(), PipelineAggregator.Parser.BUCKETS_PATH); + } + + TestPipelineAggregationBuilder(String name, String bucketsPath) { + super(name, "test", new String[] { bucketsPath }); + } + /** * Read from a stream. */ diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java index 76b8380101d5b..e47e1c3ab9748 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java @@ -57,7 +57,6 @@ import org.elasticsearch.search.aggregations.metrics.MinTests; import org.elasticsearch.search.aggregations.metrics.SumTests; import org.elasticsearch.search.aggregations.pipeline.InternalBucketMetricValueTests; -import org.elasticsearch.search.aggregations.pipeline.InternalDerivativeTests; import org.elasticsearch.search.aggregations.pipeline.InternalExtendedStatsBucketTests; import org.elasticsearch.search.aggregations.pipeline.InternalPercentilesBucketTests; import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValueTests; @@ -107,7 +106,6 @@ public class AggregationsTests extends ESTestCase { new SumTests(), new InternalValueCountTests(), new InternalSimpleValueTests(), - new InternalDerivativeTests(), new InternalBucketMetricValueTests(), new InternalStatsTests(), new InternalStatsBucketTests(), diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java index ca83722501f04..ffed9ea6ef8a4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java @@ -124,12 +124,10 @@ import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.WeightedAvgAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.InternalBucketMetricValue; import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; import org.elasticsearch.search.aggregations.pipeline.ParsedBucketMetricValue; -import org.elasticsearch.search.aggregations.pipeline.ParsedDerivative; import org.elasticsearch.search.aggregations.pipeline.ParsedExtendedStatsBucket; import org.elasticsearch.search.aggregations.pipeline.ParsedPercentilesBucket; import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue; @@ -262,7 +260,6 @@ public AggregationReduceContext forFinalReduction() { map.put(WeightedAvgAggregationBuilder.NAME, (p, c) -> ParsedWeightedAvg.fromXContent(p, (String) c)); map.put(ValueCountAggregationBuilder.NAME, (p, c) -> ParsedValueCount.fromXContent(p, (String) c)); map.put(InternalSimpleValue.NAME, (p, c) -> ParsedSimpleValue.fromXContent(p, (String) c)); - map.put(DerivativePipelineAggregationBuilder.NAME, (p, c) -> ParsedDerivative.fromXContent(p, (String) c)); map.put(InternalBucketMetricValue.NAME, (p, c) -> ParsedBucketMetricValue.fromXContent(p, (String) c)); map.put(StatsAggregationBuilder.NAME, (p, c) -> ParsedStats.fromXContent(p, (String) c)); map.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java index cb5f5882a6a1e..fb08c0286fe56 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java @@ -13,6 +13,8 @@ import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.aggregations.AggregationsPlugin; +import org.elasticsearch.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; @@ -30,7 +32,6 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.search.aggregations.PipelineAggregatorBuilders; import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.composite.DateHistogramValuesSourceBuilder; import org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; @@ -40,7 +41,6 @@ import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketScriptPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.test.AbstractXContentSerializingTestCase; @@ -102,13 +102,13 @@ public static DatafeedConfig createRandomizedDatafeedConfig(String jobId, String @Override protected NamedWriteableRegistry getNamedWriteableRegistry() { - SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + SearchModule searchModule = new SearchModule(Settings.EMPTY, List.of(new AggregationsPlugin())); return new NamedWriteableRegistry(searchModule.getNamedWriteables()); } @Override protected NamedXContentRegistry xContentRegistry() { - SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + SearchModule searchModule = new SearchModule(Settings.EMPTY, List.of(new AggregationsPlugin())); return new NamedXContentRegistry(searchModule.getNamedXContents()); } @@ -774,19 +774,18 @@ public void testDefaultFrequency_GivenAggregationsWithHistogramOrCompositeInterv } public void testSerializationOfComplexAggs() throws IOException { - MaxAggregationBuilder maxTime = AggregationBuilders.max("timestamp").field("timestamp"); - AvgAggregationBuilder avgAggregationBuilder = AggregationBuilders.avg("bytes_in_avg").field("system.network.in.bytes"); - DerivativePipelineAggregationBuilder derivativePipelineAggregationBuilder = PipelineAggregatorBuilders.derivative( + MaxAggregationBuilder maxTime = new MaxAggregationBuilder("timestamp").field("timestamp"); + AvgAggregationBuilder avgAggregationBuilder = new AvgAggregationBuilder("bytes_in_avg").field("system.network.in.bytes"); + DerivativePipelineAggregationBuilder derivativePipelineAggregationBuilder = new DerivativePipelineAggregationBuilder( "bytes_in_derivative", "bytes_in_avg" ); - BucketScriptPipelineAggregationBuilder bucketScriptPipelineAggregationBuilder = PipelineAggregatorBuilders.bucketScript( + BucketScriptPipelineAggregationBuilder bucketScriptPipelineAggregationBuilder = new BucketScriptPipelineAggregationBuilder( "non_negative_bytes", Collections.singletonMap("bytes", "bytes_in_derivative"), new Script("params.bytes > 0 ? params.bytes : null") ); - DateHistogramAggregationBuilder dateHistogram = AggregationBuilders.dateHistogram("histogram_buckets") - .field("timestamp") + DateHistogramAggregationBuilder dateHistogram = new DateHistogramAggregationBuilder("histogram_buckets").field("timestamp") .fixedInterval(new DateHistogramInterval("300000ms")) .timeZone(ZoneOffset.UTC) .subAggregation(maxTime) @@ -815,11 +814,9 @@ public void testSerializationOfComplexAggs() throws IOException { assertEquals(aggBuilder, parsedDatafeedConfig.getParsedAggregations(xContentRegistry())); assertEquals(datafeedConfig.getQuery(), parsedDatafeedConfig.getQuery()); - SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); - NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables()); try (BytesStreamOutput output = new BytesStreamOutput()) { datafeedConfig.writeTo(output); - try (StreamInput streamInput = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { + try (StreamInput streamInput = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), getNamedWriteableRegistry())) { DatafeedConfig streamedDatafeedConfig = new DatafeedConfig(streamInput); assertEquals(datafeedConfig, streamedDatafeedConfig); @@ -831,19 +828,18 @@ public void testSerializationOfComplexAggs() throws IOException { } public void testSerializationOfComplexAggsBetweenVersions() throws IOException { - MaxAggregationBuilder maxTime = AggregationBuilders.max("timestamp").field("timestamp"); - AvgAggregationBuilder avgAggregationBuilder = AggregationBuilders.avg("bytes_in_avg").field("system.network.in.bytes"); - DerivativePipelineAggregationBuilder derivativePipelineAggregationBuilder = PipelineAggregatorBuilders.derivative( + MaxAggregationBuilder maxTime = new MaxAggregationBuilder("timestamp").field("timestamp"); + AvgAggregationBuilder avgAggregationBuilder = new AvgAggregationBuilder("bytes_in_avg").field("system.network.in.bytes"); + DerivativePipelineAggregationBuilder derivativePipelineAggregationBuilder = new DerivativePipelineAggregationBuilder( "bytes_in_derivative", "bytes_in_avg" ); - BucketScriptPipelineAggregationBuilder bucketScriptPipelineAggregationBuilder = PipelineAggregatorBuilders.bucketScript( + BucketScriptPipelineAggregationBuilder bucketScriptPipelineAggregationBuilder = new BucketScriptPipelineAggregationBuilder( "non_negative_bytes", Collections.singletonMap("bytes", "bytes_in_derivative"), new Script("params.bytes > 0 ? params.bytes : null") ); - DateHistogramAggregationBuilder dateHistogram = AggregationBuilders.dateHistogram("histogram_buckets") - .field("timestamp") + DateHistogramAggregationBuilder dateHistogram = new DateHistogramAggregationBuilder("histogram_buckets").field("timestamp") .fixedInterval(new DateHistogramInterval("30000ms")) .timeZone(ZoneOffset.UTC) .subAggregation(maxTime) @@ -862,13 +858,10 @@ public void testSerializationOfComplexAggsBetweenVersions() throws IOException { ); DatafeedConfig datafeedConfig = datafeedConfigBuilder.build(); - SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); - NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables()); - try (BytesStreamOutput output = new BytesStreamOutput()) { output.setVersion(Version.CURRENT); datafeedConfig.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), getNamedWriteableRegistry())) { in.setVersion(Version.CURRENT); DatafeedConfig streamedDatafeedConfig = new DatafeedConfig(in); assertEquals(datafeedConfig, streamedDatafeedConfig); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java index d952b57ac1ac8..4df9d6be1fa01 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java @@ -10,6 +10,8 @@ import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.aggregations.AggregationsPlugin; +import org.elasticsearch.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -29,13 +31,11 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.search.aggregations.PipelineAggregatorBuilders; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketScriptPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.test.AbstractXContentSerializingTestCase; @@ -166,13 +166,13 @@ protected DatafeedUpdate doParseInstance(XContentParser parser) { @Override protected NamedWriteableRegistry getNamedWriteableRegistry() { - SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + SearchModule searchModule = new SearchModule(Settings.EMPTY, List.of(new AggregationsPlugin())); return new NamedWriteableRegistry(searchModule.getNamedWriteables()); } @Override protected NamedXContentRegistry xContentRegistry() { - SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + SearchModule searchModule = new SearchModule(Settings.EMPTY, List.of(new AggregationsPlugin())); return new NamedXContentRegistry(searchModule.getNamedXContents()); } @@ -355,19 +355,18 @@ public void testApply_GivenRandomUpdates_AssertImmutability() { } public void testSerializationOfComplexAggsBetweenVersions() throws IOException { - MaxAggregationBuilder maxTime = AggregationBuilders.max("timestamp").field("timestamp"); - AvgAggregationBuilder avgAggregationBuilder = AggregationBuilders.avg("bytes_in_avg").field("system.network.in.bytes"); - DerivativePipelineAggregationBuilder derivativePipelineAggregationBuilder = PipelineAggregatorBuilders.derivative( + MaxAggregationBuilder maxTime = new MaxAggregationBuilder("timestamp").field("timestamp"); + AvgAggregationBuilder avgAggregationBuilder = new AvgAggregationBuilder("bytes_in_avg").field("system.network.in.bytes"); + DerivativePipelineAggregationBuilder derivativePipelineAggregationBuilder = new DerivativePipelineAggregationBuilder( "bytes_in_derivative", "bytes_in_avg" ); - BucketScriptPipelineAggregationBuilder bucketScriptPipelineAggregationBuilder = PipelineAggregatorBuilders.bucketScript( + BucketScriptPipelineAggregationBuilder bucketScriptPipelineAggregationBuilder = new BucketScriptPipelineAggregationBuilder( "non_negative_bytes", Collections.singletonMap("bytes", "bytes_in_derivative"), new Script("params.bytes > 0 ? params.bytes : null") ); - DateHistogramAggregationBuilder dateHistogram = AggregationBuilders.dateHistogram("histogram_buckets") - .field("timestamp") + DateHistogramAggregationBuilder dateHistogram = new DateHistogramAggregationBuilder("histogram_buckets").field("timestamp") .fixedInterval(new DateHistogramInterval("300000ms")) .timeZone(ZoneOffset.UTC) .subAggregation(maxTime) @@ -390,13 +389,10 @@ public void testSerializationOfComplexAggsBetweenVersions() throws IOException { ); DatafeedUpdate datafeedUpdate = datafeedUpdateBuilder.build(); - SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); - NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables()); - try (BytesStreamOutput output = new BytesStreamOutput()) { output.setVersion(Version.CURRENT); datafeedUpdate.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), getNamedWriteableRegistry())) { in.setVersion(Version.CURRENT); DatafeedUpdate streamedDatafeedUpdate = new DatafeedUpdate(in); assertEquals(datafeedUpdate, streamedDatafeedUpdate);