From 3c70247cd1fa7d1118085911b9c5da571cb027ba Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 15 Oct 2019 11:53:15 +0200 Subject: [PATCH] Add Cumulative Cardinality agg (and Data Science plugin) Addresses elastic/elasticsearch#43661 --- src/Nest/Aggregations/AggregateDictionary.cs | 2 + src/Nest/Aggregations/AggregationContainer.cs | 13 +++ .../CumulativeCardinalityAggregation.cs | 23 +++++ .../Visitor/AggregationVisitor.cs | 4 + .../Aggregations/Visitor/AggregationWalker.cs | 1 + .../XPack/Info/XPackInfo/XPackInfoResponse.cs | 3 + .../Info/XPackUsage/XPackUsageResponse.cs | 3 + ...ulativeCardinalityAggregationUsageTests.cs | 85 +++++++++++++++++++ .../CumulativeSumAggregationUsageTests.cs | 2 +- 9 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 src/Nest/Aggregations/Pipeline/CumulativeCardinality/CumulativeCardinalityAggregation.cs create mode 100644 src/Tests/Tests/Aggregations/Pipeline/CumulativeCardinality/CumulativeCardinalityAggregationUsageTests.cs diff --git a/src/Nest/Aggregations/AggregateDictionary.cs b/src/Nest/Aggregations/AggregateDictionary.cs index 5f1cf895d88..ece998c0fa3 100644 --- a/src/Nest/Aggregations/AggregateDictionary.cs +++ b/src/Nest/Aggregations/AggregateDictionary.cs @@ -55,6 +55,8 @@ internal static string[] TypedKeyTokens(string key) public ValueAggregate CumulativeSum(string key) => TryGet(key); + public ValueAggregate CumulativeCardinality(string key) => TryGet(key); + public ValueAggregate BucketScript(string key) => TryGet(key); public ValueAggregate SerialDifferencing(string key) => TryGet(key); diff --git a/src/Nest/Aggregations/AggregationContainer.cs b/src/Nest/Aggregations/AggregationContainer.cs index ecadb8dd8e4..b9b68496bd8 100644 --- a/src/Nest/Aggregations/AggregationContainer.cs +++ b/src/Nest/Aggregations/AggregationContainer.cs @@ -113,6 +113,9 @@ public interface IAggregationContainer [DataMember(Name = "cumulative_sum")] ICumulativeSumAggregation CumulativeSum { get; set; } + [DataMember(Name = "cumulative_cardinality")] + ICumulativeCardinalityAggregation CumulativeCardinality { get; set; } + [DataMember(Name = "date_histogram")] IDateHistogramAggregation DateHistogram { get; set; } @@ -282,6 +285,8 @@ public class AggregationContainer : IAggregationContainer public ICumulativeSumAggregation CumulativeSum { get; set; } + public ICumulativeCardinalityAggregation CumulativeCardinality { get; set; } + public IDateHistogramAggregation DateHistogram { get; set; } public IAutoDateHistogramAggregation AutoDateHistogram { get; set; } @@ -358,6 +363,7 @@ public class AggregationContainer : IAggregationContainer public IStatsAggregation Stats { get; set; } public IStatsBucketAggregation StatsBucket { get; set; } + public ISumAggregation Sum { get; set; } public ISumBucketAggregation SumBucket { get; set; } @@ -425,6 +431,8 @@ public class AggregationContainerDescriptor : DescriptorBase selector ) => _SetInnerAggregation(name, selector, (a, d) => a.CumulativeSum = d); + public AggregationContainerDescriptor CumulativeCardinality(string name, + Func selector + ) => + _SetInnerAggregation(name, selector, (a, d) => a.CumulativeCardinality = d); + public AggregationContainerDescriptor SerialDifferencing(string name, Func selector ) => diff --git a/src/Nest/Aggregations/Pipeline/CumulativeCardinality/CumulativeCardinalityAggregation.cs b/src/Nest/Aggregations/Pipeline/CumulativeCardinality/CumulativeCardinalityAggregation.cs new file mode 100644 index 00000000000..01653bd888e --- /dev/null +++ b/src/Nest/Aggregations/Pipeline/CumulativeCardinality/CumulativeCardinalityAggregation.cs @@ -0,0 +1,23 @@ +using Elasticsearch.Net.Utf8Json; + +namespace Nest +{ + [InterfaceDataContract] + [ReadAs(typeof(CumulativeCardinalityAggregation))] + public interface ICumulativeCardinalityAggregation : IPipelineAggregation { } + + public class CumulativeCardinalityAggregation + : PipelineAggregationBase, ICumulativeCardinalityAggregation + { + internal CumulativeCardinalityAggregation() { } + + public CumulativeCardinalityAggregation(string name, SingleBucketsPath bucketsPath) + : base(name, bucketsPath) { } + + internal override void WrapInContainer(AggregationContainer c) => c.CumulativeCardinality = this; + } + + public class CumulativeCardinalityAggregationDescriptor + : PipelineAggregationDescriptorBase + , ICumulativeCardinalityAggregation { } +} diff --git a/src/Nest/Aggregations/Visitor/AggregationVisitor.cs b/src/Nest/Aggregations/Visitor/AggregationVisitor.cs index 0b60e2f617a..d6e2bdc2027 100644 --- a/src/Nest/Aggregations/Visitor/AggregationVisitor.cs +++ b/src/Nest/Aggregations/Visitor/AggregationVisitor.cs @@ -110,6 +110,8 @@ public interface IAggregationVisitor void Visit(ICumulativeSumAggregation aggregation); + void Visit(ICumulativeCardinalityAggregation aggregation); + void Visit(ISerialDifferencingAggregation aggregation); void Visit(IBucketScriptAggregation aggregation); @@ -179,6 +181,8 @@ public virtual void Visit(IPercentilesBucketAggregation aggregation) { } public virtual void Visit(ICumulativeSumAggregation aggregation) { } + public virtual void Visit(ICumulativeCardinalityAggregation aggregation) { } + public virtual void Visit(IBucketScriptAggregation aggregation) { } public virtual void Visit(ISamplerAggregation aggregation) { } diff --git a/src/Nest/Aggregations/Visitor/AggregationWalker.cs b/src/Nest/Aggregations/Visitor/AggregationWalker.cs index 510ff903bac..aea1e1a7862 100644 --- a/src/Nest/Aggregations/Visitor/AggregationWalker.cs +++ b/src/Nest/Aggregations/Visitor/AggregationWalker.cs @@ -48,6 +48,7 @@ public void Walk(IAggregationContainer aggregation, IAggregationVisitor visitor) Accept(v, d.Aggregations); }); AcceptAggregation(aggregation.CumulativeSum, visitor, (v, d) => v.Visit(d)); + AcceptAggregation(aggregation.CumulativeCardinality, visitor, (v, d) => v.Visit(d)); AcceptAggregation(aggregation.DateHistogram, visitor, (v, d) => { v.Visit(d); diff --git a/src/Nest/XPack/Info/XPackInfo/XPackInfoResponse.cs b/src/Nest/XPack/Info/XPackInfo/XPackInfoResponse.cs index ef87b6c83f7..05908f5e007 100644 --- a/src/Nest/XPack/Info/XPackInfo/XPackInfoResponse.cs +++ b/src/Nest/XPack/Info/XPackInfo/XPackInfoResponse.cs @@ -53,6 +53,9 @@ public class XPackFeatures [DataMember(Name = "flattened")] public XPackFeature Flattened { get; internal set; } + [DataMember(Name = "data_science")] + public XPackFeature DataScience { get; internal set; } + [DataMember(Name = "graph")] public XPackFeature Graph { get; internal set; } diff --git a/src/Nest/XPack/Info/XPackUsage/XPackUsageResponse.cs b/src/Nest/XPack/Info/XPackUsage/XPackUsageResponse.cs index fde7e1347b2..bf23f5e29be 100644 --- a/src/Nest/XPack/Info/XPackUsage/XPackUsageResponse.cs +++ b/src/Nest/XPack/Info/XPackUsage/XPackUsageResponse.cs @@ -51,6 +51,9 @@ public class XPackUsageResponse : ResponseBase [DataMember(Name = "flattened")] public XPackUsage Flattened { get; internal set; } + [DataMember(Name = "data_science")] + public XPackUsage DataScience { get; internal set; } + [DataMember(Name = "ilm")] public IlmUsage IndexLifecycleManagement { get; internal set; } diff --git a/src/Tests/Tests/Aggregations/Pipeline/CumulativeCardinality/CumulativeCardinalityAggregationUsageTests.cs b/src/Tests/Tests/Aggregations/Pipeline/CumulativeCardinality/CumulativeCardinalityAggregationUsageTests.cs new file mode 100644 index 00000000000..6f8d4d318be --- /dev/null +++ b/src/Tests/Tests/Aggregations/Pipeline/CumulativeCardinality/CumulativeCardinalityAggregationUsageTests.cs @@ -0,0 +1,85 @@ +using System; +using FluentAssertions; +using Nest; +using Tests.Core.Extensions; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Domain; +using Tests.Framework.EndpointTests.TestState; + +namespace Tests.Aggregations.Pipeline.CumulativeCardinality +{ + public class CumulativeCardinalityAggregationUsageTests : AggregationUsageTestBase + { + public CumulativeCardinalityAggregationUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override object AggregationJson => new + { + projects_started_per_month = new + { + date_histogram = new + { + field = "startedOn", + calendar_interval = "month", + }, + aggs = new + { + commits = new + { + cardinality = new + { + field = "numberOfCommits" + } + }, + cumulative_commits = new + { + cumulative_cardinality = new + { + buckets_path = "commits" + } + } + } + } + }; + + protected override Func, IAggregationContainer> FluentAggs => a => a + .DateHistogram("projects_started_per_month", dh => dh + .Field(p => p.StartedOn) + .CalendarInterval(DateInterval.Month) + .Aggregations(aa => aa + .Cardinality("commits", sm => sm + .Field(p => p.NumberOfCommits) + ) + .CumulativeCardinality("cumulative_commits", d => d + .BucketsPath("commits") + ) + ) + ); + + protected override AggregationDictionary InitializerAggs => + new DateHistogramAggregation("projects_started_per_month") + { + Field = "startedOn", + CalendarInterval = DateInterval.Month, + Aggregations = + new CardinalityAggregation("commits", "numberOfCommits") && + new CumulativeCardinalityAggregation("cumulative_commits", "commits") + }; + + protected override void ExpectResponse(ISearchResponse response) + { + response.ShouldBeValid(); + + var projectsPerMonth = response.Aggregations.DateHistogram("projects_started_per_month"); + projectsPerMonth.Should().NotBeNull(); + projectsPerMonth.Buckets.Should().NotBeNull(); + projectsPerMonth.Buckets.Count.Should().BeGreaterThan(0); + + foreach (var item in projectsPerMonth.Buckets) + { + var commitsDerivative = item.CumulativeCardinality("cumulative_commits"); + commitsDerivative.Should().NotBeNull(); + commitsDerivative.Value.Should().NotBe(null); + } + } + } +} diff --git a/src/Tests/Tests/Aggregations/Pipeline/CumulativeSum/CumulativeSumAggregationUsageTests.cs b/src/Tests/Tests/Aggregations/Pipeline/CumulativeSum/CumulativeSumAggregationUsageTests.cs index 4ccaf154b7d..d2a30ce47d1 100644 --- a/src/Tests/Tests/Aggregations/Pipeline/CumulativeSum/CumulativeSumAggregationUsageTests.cs +++ b/src/Tests/Tests/Aggregations/Pipeline/CumulativeSum/CumulativeSumAggregationUsageTests.cs @@ -78,7 +78,7 @@ protected override void ExpectResponse(ISearchResponse response) foreach (var item in projectsPerMonth.Buckets) { - var commitsDerivative = item.Derivative("cumulative_commits"); + var commitsDerivative = item.CumulativeSum("cumulative_commits"); commitsDerivative.Should().NotBeNull(); commitsDerivative.Value.Should().NotBe(null); }