From a9fff181e4ea8281ad907e7b2e0d90e70013a4de Mon Sep 17 00:00:00 2001 From: Baha Aiman Date: Tue, 22 Aug 2023 07:51:16 -0700 Subject: [PATCH] feat(datastore): SUM and AVG aggregations (#8307) * feat(datastore): SUM and AVG aggregations * feat(datastore): Fixing integration tests for SUM AVG * feat(datastore): Fixing integration tests * feat(datastore): Fixing integration tests * feat(datastore): Updating protos * feat(datastore): updating protos * feat(datastore): Undo go.work.sum changes * feat(datastore): Used new protos * feat(datastore): Use latest protos --------- Co-authored-by: meredithslota Co-authored-by: kolea2 <45548808+kolea2@users.noreply.github.com> --- datastore/go.mod | 15 ++--- datastore/go.sum | 29 ++++----- datastore/integration_test.go | 111 ++++++++++++++++++++++++++++------ datastore/query.go | 44 ++++++++++++++ datastore/testdata/index.yaml | 16 ++++- 5 files changed, 174 insertions(+), 41 deletions(-) diff --git a/datastore/go.mod b/datastore/go.mod index d93c95992785..2a1afec49f32 100644 --- a/datastore/go.mod +++ b/datastore/go.mod @@ -3,20 +3,20 @@ module cloud.google.com/go/datastore go 1.19 require ( - cloud.google.com/go v0.110.2 - cloud.google.com/go/longrunning v0.5.0 + cloud.google.com/go v0.110.7 + cloud.google.com/go/longrunning v0.5.1 github.com/golang/protobuf v1.5.3 github.com/google/go-cmp v0.5.9 github.com/googleapis/gax-go/v2 v2.12.0 google.golang.org/api v0.128.0 - google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc - google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc - google.golang.org/grpc v1.56.1 + google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93 + google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 + google.golang.org/grpc v1.57.0 google.golang.org/protobuf v1.31.0 ) require ( - cloud.google.com/go/compute v1.19.3 // indirect + cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/s2a-go v0.1.4 // indirect @@ -25,9 +25,10 @@ require ( golang.org/x/crypto v0.9.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect ) diff --git a/datastore/go.sum b/datastore/go.sum index b2881fae3b35..e636509748b3 100644 --- a/datastore/go.sum +++ b/datastore/go.sum @@ -1,13 +1,13 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= -cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= -cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds= -cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= +cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= +cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/longrunning v0.5.0 h1:DK8BH0+hS+DIvc9a2TPnteUievsTCH4ORMAASSb7JcQ= -cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= +cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -118,6 +118,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -162,12 +163,12 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93 h1:zv6ieVm8jNcN33At1+APsRISkRgynuWUxUhv6G123jY= +google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -176,8 +177,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= -google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/datastore/integration_test.go b/datastore/integration_test.go index 5c873e16c2d5..6a01926449cc 100644 --- a/datastore/integration_test.go +++ b/datastore/integration_test.go @@ -37,6 +37,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" ) // TODO(djd): Make test entity clean up more robust: some test entities may @@ -52,6 +53,7 @@ var suffix string const ( replayFilename = "datastore.replay" envDatabases = "GCLOUD_TESTS_GOLANG_DATASTORE_DATABASES" + keyPrefix = "TestIntegration_" ) type replayInfo struct { @@ -471,6 +473,8 @@ func TestIntegration_NilKey(t *testing.T) { type SQChild struct { I, J int T, U int64 + V float64 + W string } type SQTestCase struct { @@ -701,17 +705,17 @@ func TestIntegration_AggregationQueries(t *testing.T) { client := newTestClient(ctx, t) defer client.Close() - parent := NameKey("SQParent", "TestIntegration_Filters"+suffix, nil) + parent := NameKey("SQParent", keyPrefix+"AggregationQueries"+suffix, nil) now := timeNow.Truncate(time.Millisecond).Unix() children := []*SQChild{ - {I: 0, T: now, U: now}, - {I: 1, T: now, U: now}, - {I: 2, T: now, U: now}, - {I: 3, T: now, U: now}, - {I: 4, T: now, U: now}, - {I: 5, T: now, U: now}, - {I: 6, T: now, U: now}, - {I: 7, T: now, U: now}, + {I: 0, T: now, U: now, V: 1.5, W: "str"}, + {I: 1, T: now, U: now, V: 1.5, W: "str"}, + {I: 2, T: now, U: now, V: 1.5, W: "str"}, + {I: 3, T: now, U: now, V: 1.5, W: "str"}, + {I: 4, T: now, U: now, V: 1.5, W: "str"}, + {I: 5, T: now, U: now, V: 1.5, W: "str"}, + {I: 6, T: now, U: now, V: 1.5, W: "str"}, + {I: 7, T: now, U: now, V: 1.5, W: "str"}, } keys := make([]*Key, len(children)) @@ -729,7 +733,6 @@ func TestIntegration_AggregationQueries(t *testing.T) { } }() - baseQuery := NewQuery("SQChild").Ancestor(parent) testCases := []struct { desc string aggQuery *AggregationQuery @@ -738,21 +741,91 @@ func TestIntegration_AggregationQueries(t *testing.T) { wantAggResult AggregationResult }{ { - desc: "Count Failure - Missing index", - aggQuery: baseQuery.Filter("T>=", now).NewAggregationQuery().WithCount("count"), - wantFailure: true, - wantErrMsg: "no matching index found", - wantAggResult: nil, + desc: "Count Failure - Missing index", + aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T>=", now). + NewAggregationQuery(). + WithCount("count"), + wantFailure: true, + wantErrMsg: "no matching index found", }, { - desc: "Count Success", - aggQuery: baseQuery.Filter("T=", now).Filter("I>=", 3).NewAggregationQuery().WithCount("count"), - wantFailure: false, - wantErrMsg: "", + desc: "Count Success", + aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Filter("I>=", 3). + NewAggregationQuery(). + WithCount("count"), wantAggResult: map[string]interface{}{ "count": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: 5}}, }, }, + { + desc: "Multiple aggregations", + aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now). + NewAggregationQuery(). + WithSum("I", "i_sum"). + WithAvg("I", "avg"). + WithSum("V", "v_sum"), + wantAggResult: map[string]interface{}{ + "i_sum": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: 28}}, + "v_sum": &pb.Value{ValueType: &pb.Value_DoubleValue{DoubleValue: 12}}, + "avg": &pb.Value{ValueType: &pb.Value_DoubleValue{DoubleValue: 3.5}}, + }, + }, + { + desc: "Multiple aggregations with limit ", + aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Limit(2). + NewAggregationQuery(). + WithSum("I", "sum"). + WithAvg("I", "avg"), + wantAggResult: map[string]interface{}{ + "sum": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: 1}}, + "avg": &pb.Value{ValueType: &pb.Value_DoubleValue{DoubleValue: 0.5}}, + }, + }, + { + desc: "Multiple aggregations on non-numeric field", + aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Limit(2). + NewAggregationQuery(). + WithSum("W", "sum"). + WithAvg("W", "avg"), + wantAggResult: map[string]interface{}{ + "sum": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: int64(0)}}, + "avg": &pb.Value{ValueType: &pb.Value_NullValue{NullValue: structpb.NullValue_NULL_VALUE}}, + }, + }, + { + desc: "Sum aggregation without alias", + aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now). + NewAggregationQuery(). + WithSum("I", ""), + wantAggResult: map[string]interface{}{ + "property_1": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: 28}}, + }, + }, + { + desc: "Average aggregation without alias", + aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now). + NewAggregationQuery(). + WithAvg("I", ""), + wantAggResult: map[string]interface{}{ + "property_1": &pb.Value{ValueType: &pb.Value_DoubleValue{DoubleValue: 3.5}}, + }, + }, + { + desc: "Sum aggregation on '__key__'", + aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now). + NewAggregationQuery(). + WithSum("__key__", ""), + wantFailure: true, + wantErrMsg: "Aggregations are not supported for the property", + }, + { + desc: "Average aggregation on '__key__'", + aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now). + NewAggregationQuery(). + WithAvg("__key__", ""), + wantFailure: true, + wantErrMsg: "Aggregations are not supported for the property", + }, } for _, testCase := range testCases { diff --git a/datastore/query.go b/datastore/query.go index 702dce38a873..c9f9163c310d 100644 --- a/datastore/query.go +++ b/datastore/query.go @@ -1057,5 +1057,49 @@ func (aq *AggregationQuery) WithCount(alias string) *AggregationQuery { return aq } +// WithSum specifies that the aggregation query should provide a sum of the values +// of the provided field in the results returned by the underlying Query. +// The alias argument can be empty or a valid Datastore entity property name. It can be used +// as key in the AggregationResult to get the sum value. If alias is empty, Datastore +// will autogenerate a key. +func (aq *AggregationQuery) WithSum(fieldName string, alias string) *AggregationQuery { + aqpb := &pb.AggregationQuery_Aggregation{ + Alias: alias, + Operator: &pb.AggregationQuery_Aggregation_Sum_{ + Sum: &pb.AggregationQuery_Aggregation_Sum{ + Property: &pb.PropertyReference{ + Name: fieldName, + }, + }, + }, + } + + aq.aggregationQueries = append(aq.aggregationQueries, aqpb) + + return aq +} + +// WithAvg specifies that the aggregation query should provide an average of the values +// of the provided field in the results returned by the underlying Query. +// The alias argument can be empty or a valid Datastore entity property name. It can be used +// as key in the AggregationResult to get the sum value. If alias is empty, Datastore +// will autogenerate a key. +func (aq *AggregationQuery) WithAvg(fieldName string, alias string) *AggregationQuery { + aqpb := &pb.AggregationQuery_Aggregation{ + Alias: alias, + Operator: &pb.AggregationQuery_Aggregation_Avg_{ + Avg: &pb.AggregationQuery_Aggregation_Avg{ + Property: &pb.PropertyReference{ + Name: fieldName, + }, + }, + }, + } + + aq.aggregationQueries = append(aq.aggregationQueries, aqpb) + + return aq +} + // AggregationResult contains the results of an aggregation query. type AggregationResult map[string]interface{} diff --git a/datastore/testdata/index.yaml b/datastore/testdata/index.yaml index 47bc9de867f4..150fd2c448bf 100644 --- a/datastore/testdata/index.yaml +++ b/datastore/testdata/index.yaml @@ -38,4 +38,18 @@ indexes: properties: - name: T - name: J - - name: U \ No newline at end of file + - name: U + +- kind: SQChild + ancestor: yes + properties: + - name: T + - name: I + - name: V + +- kind: SQChild + ancestor: yes + properties: + - name: T + - name: W + \ No newline at end of file