From 56b930a85294f21e61e79c89caaa43f8c99c378c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 19 Mar 2023 22:08:14 +0000 Subject: [PATCH 1/9] fix(deps): update module github.com/goccy/go-json to v0.10.2 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2a3cd3c7..a4a9b586 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/go-playground/validator/v10 v10.11.2 github.com/go-redis/redis/v8 v8.11.5 github.com/go-redsync/redsync/v4 v4.8.1 - github.com/goccy/go-json v0.10.0 + github.com/goccy/go-json v0.10.2 github.com/gofiber/contrib/fibersentry v1.0.2 github.com/gofiber/contrib/otelfiber v1.0.3 github.com/gofiber/fiber/v2 v2.42.0 diff --git a/go.sum b/go.sum index 4263cb24..2d7731ae 100644 --- a/go.sum +++ b/go.sum @@ -179,6 +179,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofiber/adaptor/v2 v2.1.31 h1:E7LJre4uBc+RDsQfHCE+LKVkFcciSMYu4KhzbvoWgKU= github.com/gofiber/adaptor/v2 v2.1.31/go.mod h1:vdSG9JhOhOLYjE4j14fx6sJvLJNFVf9o6rSyB5GkU4s= From b5cb713c20b7276d61627a6e56490fb020fb09bf Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 4 Apr 2023 21:43:33 +0000 Subject: [PATCH 2/9] chore(deps): update golang docker tag to v1.20.3 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 98e72769..646c1a92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.2-alpine AS base +FROM golang:1.20.3-alpine AS base WORKDIR /app # builder From 75637efe57f8167643e34f7dd8a5041819ef1188 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 6 Apr 2023 14:36:00 -0400 Subject: [PATCH 3/9] fix: advanced query validate endtime > starttime --- internal/controller/v2/result.go | 2 +- internal/model/types/advanced_query.go | 2 +- internal/util/validator.go | 9 ++++++++ test/api_test.go | 27 ++++++++++++++++++++++ test/v2_advanced_query_test.go | 31 +++++++++++++++++++++++++ test/v2_report_test.go | 32 ++------------------------ 6 files changed, 71 insertions(+), 32 deletions(-) create mode 100644 test/v2_advanced_query_test.go diff --git a/internal/controller/v2/result.go b/internal/controller/v2/result.go index 565db16d..177b6ea7 100644 --- a/internal/controller/v2/result.go +++ b/internal/controller/v2/result.go @@ -235,7 +235,7 @@ func (c *Result) GetTrends(ctx *fiber.Ctx) error { // @Success 200 {object} modelv2.AdvancedQueryResult{advanced_results=[]modelv2.DropMatrixQueryResult} "Drop Matrix Response: when `interval` has been left undefined." // @Success 202 {object} modelv2.AdvancedQueryResult{advanced_results=[]modelv2.TrendQueryResult} "Trend Response: when `interval` has been defined a value greater than `0`. Notice that this response still responds with a status code of `200`, but due to swagger limitations, to denote a different response with the same status code is not possible. Therefore, a status code of `202` is used, only for the purpose of workaround." // @Failure 500 {object} pgerr.PenguinError "An unexpected error occurred" -// @Router /PenguinStats/api/v2/advanced [POST] +// @Router /PenguinStats/api/v2/result/advanced [POST] func (c *Result) AdvancedQuery(ctx *fiber.Ctx) error { var request types.AdvancedQueryRequest if err := rekuest.ValidBody(ctx, &request); err != nil { diff --git a/internal/model/types/advanced_query.go b/internal/model/types/advanced_query.go index ad389d79..1bd6f215 100644 --- a/internal/model/types/advanced_query.go +++ b/internal/model/types/advanced_query.go @@ -13,6 +13,6 @@ type AdvancedQuery struct { IsPersonal null.Bool `json:"isPersonal" swaggertype:"boolean"` SourceCategory string `json:"sourceCategory" validate:"sourcecategory"` StartTime null.Int `json:"start" swaggertype:"integer"` - EndTime null.Int `json:"end" swaggertype:"integer"` + EndTime null.Int `json:"end" validate:"gtfield=StartTime" swaggertype:"integer"` Interval null.Int `json:"interval" swaggertype:"integer"` } diff --git a/internal/util/validator.go b/internal/util/validator.go index 3755aed5..623a2c71 100644 --- a/internal/util/validator.go +++ b/internal/util/validator.go @@ -22,6 +22,7 @@ func NewValidator() *validator.Validate { validate.RegisterValidation("semverprefixed", semverPrefixed) validate.RegisterValidation("arkserver", arkServer) validate.RegisterValidation("sourcecategory", sourceCategory) + validate.RegisterCustomTypeFunc(nullIntValuer, null.Int{}) validate.RegisterCustomTypeFunc(nullStringValuer, null.String{}) return validate @@ -55,6 +56,14 @@ func sourceCategory(fl validator.FieldLevel) bool { return val == "" || val == constant.SourceCategoryAll || val == constant.SourceCategoryAutomated || val == constant.SourceCategoryManual } +func nullIntValuer(field reflect.Value) interface{} { + if valuer, ok := field.Interface().(null.Int); ok { + return valuer.Int64 + } + + return nil +} + func nullStringValuer(field reflect.Value) interface{} { if valuer, ok := field.Interface().(null.String); ok { return valuer.String diff --git a/test/api_test.go b/test/api_test.go index b745333e..b8626408 100644 --- a/test/api_test.go +++ b/test/api_test.go @@ -1,6 +1,8 @@ package test import ( + "bytes" + "io" "net/http" "net/http/httptest" "sync" @@ -8,6 +10,7 @@ import ( "github.com/gofiber/fiber/v2" "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" "go.uber.org/fx" "go.uber.org/fx/fxtest" @@ -52,6 +55,30 @@ func request(t *testing.T, req *http.Request, msTimeout ...int) *http.Response { return resp } +func JsonRequestCustom(t *testing.T, req *http.Request) (*http.Response, *gjson.Result) { + t.Helper() + + resp := request(t, req) + + bodyBytes, err := io.ReadAll(resp.Body) + assert.NoError(t, err, "failed to read response body") + + body := gjson.ParseBytes(bodyBytes) + + return resp, &body +} + +func JsonRequest(t *testing.T, path, body string, headers *http.Header) (*http.Response, *gjson.Result) { + t.Helper() + + req := httptest.NewRequest(http.MethodPost, path, bytes.NewBufferString(body)) + if headers != nil { + req.Header = *headers + } + req.Header.Set("Content-Type", "application/json") + return JsonRequestCustom(t, req) +} + func TestAPIMeta(t *testing.T) { startup(t) t.Parallel() diff --git a/test/v2_advanced_query_test.go b/test/v2_advanced_query_test.go new file mode 100644 index 00000000..0eb56cf7 --- /dev/null +++ b/test/v2_advanced_query_test.go @@ -0,0 +1,31 @@ +package test + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" +) + +func TestAPIV2AdvancedQuery(t *testing.T) { + startup(t) + t.Parallel() + + query := func(body string, headers *http.Header) (*http.Response, *gjson.Result) { + t.Helper() + return JsonRequest(t, "/PenguinStats/api/v2/result/advanced", body, headers) + } + + t.Run("valid body", func(t *testing.T) { + h, j := query(`{"queries":[{"stageId":"main_01-07","itemIds":[],"server":"CN","isPersonal":false,"sourceCategory":"all","start":1556668800000,"end":1562630400000}]}`, nil) + assert.Equal(t, http.StatusOK, h.StatusCode) + assert.NotEmpty(t, len(j.Get("advanced_results").String())) + }) + + t.Run("invalid body", func(t *testing.T) { + h, j := query(`{"queries":[{"stageId":"main_01-07","itemIds":[],"server":"CN","isPersonal":false,"sourceCategory":"all","start":1556668800000,"end":1556668800000}]}`, nil) + assert.Equal(t, http.StatusBadRequest, h.StatusCode) + assert.Equal(t, "invalid time range", j.Get("error").String()) + }) +} diff --git a/test/v2_report_test.go b/test/v2_report_test.go index 119d5ca7..ef614c46 100644 --- a/test/v2_report_test.go +++ b/test/v2_report_test.go @@ -1,10 +1,7 @@ package test import ( - "bytes" - "io" "net/http" - "net/http/httptest" "testing" "time" @@ -17,39 +14,14 @@ func TestAPIV2Reports(t *testing.T) { startup(t) t.Parallel() - // helpers - jsonReqCustom := func(req *http.Request) (*http.Response, *gjson.Result) { - t.Helper() - - resp := request(t, req) - - bodyBytes, err := io.ReadAll(resp.Body) - assert.NoError(t, err, "failed to read response body") - - body := gjson.ParseBytes(bodyBytes) - - return resp, &body - } - - jsonReq := func(path, body string, headers *http.Header) (*http.Response, *gjson.Result) { - t.Helper() - - req := httptest.NewRequest(http.MethodPost, path, bytes.NewBufferString(body)) - if headers != nil { - req.Header = *headers - } - req.Header.Set("Content-Type", "application/json") - return jsonReqCustom(req) - } - report := func(body string, headers *http.Header) (*http.Response, *gjson.Result) { t.Helper() - return jsonReq("/PenguinStats/api/v2/report", body, headers) + return JsonRequest(t, "/PenguinStats/api/v2/report", body, headers) } recall := func(body string, headers *http.Header) (*http.Response, *gjson.Result) { t.Helper() - return jsonReq("/PenguinStats/api/v2/report/recall", body, headers) + return JsonRequest(t, "/PenguinStats/api/v2/report/recall", body, headers) } // tests From aabda1ab76a583d91c0cfc15a59b096961a1665f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 5 Apr 2023 04:05:14 +0000 Subject: [PATCH 4/9] fix(deps): update module github.com/go-redis/redis/v8 to v9 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a4a9b586..7fa1b738 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/go-playground/locales v0.14.1 github.com/go-playground/universal-translator v0.18.1 github.com/go-playground/validator/v10 v10.11.2 - github.com/go-redis/redis/v8 v8.11.5 + github.com/go-redis/redis/v9 v9.0.3 github.com/go-redsync/redsync/v4 v4.8.1 github.com/goccy/go-json v0.10.2 github.com/gofiber/contrib/fibersentry v1.0.2 From 3774c70ab7c587d3d135b6218e09d32d8ace1835 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 6 Apr 2023 14:41:35 -0400 Subject: [PATCH 5/9] feat: upgrade to redis/go-redis/v9 --- go.mod | 2 +- go.sum | 15 +++++---------- internal/app/appconfig/spec.go | 2 +- internal/controller/v2/report.go | 2 +- internal/infra/redis.go | 2 +- internal/infra/redsync.go | 4 ++-- internal/pkg/fiberstore/redis.go | 2 +- internal/service/health.go | 2 +- internal/service/report.go | 2 +- internal/workers/reportwkr/reportwkr.go | 2 +- 10 files changed, 15 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 7fa1b738..c8a55344 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/go-playground/locales v0.14.1 github.com/go-playground/universal-translator v0.18.1 github.com/go-playground/validator/v10 v10.11.2 - github.com/go-redis/redis/v9 v9.0.3 github.com/go-redsync/redsync/v4 v4.8.1 github.com/goccy/go-json v0.10.2 github.com/gofiber/contrib/fibersentry v1.0.2 @@ -32,6 +31,7 @@ require ( github.com/oschwald/geoip2-golang v1.8.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 + github.com/redis/go-redis/v9 v9.0.3 github.com/rs/xid v1.4.0 github.com/rs/zerolog v1.29.0 github.com/samber/lo v1.37.0 diff --git a/go.sum b/go.sum index 2d7731ae..5c8288e1 100644 --- a/go.sum +++ b/go.sum @@ -74,7 +74,9 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.5.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w= +github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= github.com/bsm/gomega v1.20.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk= +github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -122,7 +124,6 @@ github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8Wlg github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gabstv/go-bsdiff v1.0.5 h1:g29MC/38Eaig+iAobW10/CiFvPtin8U3Jj4yNLcNG9k= github.com/gabstv/go-bsdiff v1.0.5/go.mod h1:/Zz6GK+/f/TMylRtVaW3uwZlb0FZITILfA0q12XKGwg= @@ -170,15 +171,12 @@ github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGK github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4= github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= -github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= -github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-redsync/redsync/v4 v4.8.1 h1:rq2RvdTI0obznMdxKUWGdmmulo7lS9yCzb8fgDKOlbM= github.com/go-redsync/redsync/v4 v4.8.1/go.mod h1:LmUAsQuQxhzZAoGY7JS6+dNhNmZyonMZiiEDY9plotM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= -github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -345,7 +343,6 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU= github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= @@ -353,12 +350,10 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/oschwald/geoip2-golang v1.8.0 h1:KfjYB8ojCEn/QLqsDU0AzrJ3R5Qa9vFlx3z6SLNcKTs= github.com/oschwald/geoip2-golang v1.8.0/go.mod h1:R7bRvYjOeaoenAp9sKRS8GX5bJWcZ0laWO5+DauEktw= github.com/oschwald/maxminddb-golang v1.10.0 h1:Xp1u0ZhqkSuopaKmk1WwHtjF0H9Hd9181uj2MQ5Vndg= @@ -402,8 +397,9 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE= github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= +github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k= +github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 h1:Qp27Idfgi6ACvFQat5+VJvlYToylpM/hcyLBI3WaKPA= github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052/go.mod h1:uvX/8buq8uVeiZiFht+0lqSLBHF+uGV8BrTv8W/SIwk= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -893,7 +889,6 @@ gopkg.in/guregu/null.v3 v3.5.0 h1:xTcasT8ETfMcUHn0zTvIYtQud/9Mx5dJqD554SZct0o= gopkg.in/guregu/null.v3 v3.5.0/go.mod h1:E4tX2Qe3h7QdL+uZ3a0vqvYwKQsRSQKM5V4YltdgH9Y= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/app/appconfig/spec.go b/internal/app/appconfig/spec.go index c9203bf5..87716893 100644 --- a/internal/app/appconfig/spec.go +++ b/internal/app/appconfig/spec.go @@ -55,7 +55,7 @@ type ConfigSpec struct { NatsURL string `required:"true" split_words:"true" default:"nats://127.0.0.1:4222"` // RedisURL is the URL of the Redis server, and by default uses redis db 1, to avoid potential collision - // with the previous running backend instance. See https://pkg.go.dev/github.com/go-redis/redis/v8#ParseURL + // with the previous running backend instance. See https://pkg.go.dev/github.com/redis/go-redis/v9#ParseURL // for more information on how to construct a Redis URL. RedisURL string `required:"true" split_words:"true" default:"redis://127.0.0.1:6379/1"` diff --git a/internal/controller/v2/report.go b/internal/controller/v2/report.go index a67f4a90..89b133cc 100644 --- a/internal/controller/v2/report.go +++ b/internal/controller/v2/report.go @@ -4,10 +4,10 @@ import ( "strings" "exusiai.dev/gommon/constant" - "github.com/go-redis/redis/v8" "github.com/go-redsync/redsync/v4" "github.com/goccy/go-json" "github.com/gofiber/fiber/v2" + "github.com/redis/go-redis/v9" "github.com/rs/zerolog/log" "go.uber.org/fx" diff --git a/internal/infra/redis.go b/internal/infra/redis.go index e83bc90c..d90897ae 100644 --- a/internal/infra/redis.go +++ b/internal/infra/redis.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/rs/zerolog/log" "exusiai.dev/backend-next/internal/app/appconfig" diff --git a/internal/infra/redsync.go b/internal/infra/redsync.go index ec9f2cc2..04fdca98 100644 --- a/internal/infra/redsync.go +++ b/internal/infra/redsync.go @@ -1,9 +1,9 @@ package infra import ( - goredislib "github.com/go-redis/redis/v8" "github.com/go-redsync/redsync/v4" - "github.com/go-redsync/redsync/v4/redis/goredis/v8" + "github.com/go-redsync/redsync/v4/redis/goredis/v9" + goredislib "github.com/redis/go-redis/v9" ) func RedSync(client *goredislib.Client) *redsync.Redsync { diff --git a/internal/pkg/fiberstore/redis.go b/internal/pkg/fiberstore/redis.go index 6b2e45e1..5dcf5bea 100644 --- a/internal/pkg/fiberstore/redis.go +++ b/internal/pkg/fiberstore/redis.go @@ -4,8 +4,8 @@ import ( "context" "time" - "github.com/go-redis/redis/v8" "github.com/gofiber/fiber/v2" + "github.com/redis/go-redis/v9" ) type Redis struct { diff --git a/internal/service/health.go b/internal/service/health.go index 2a4ff7c0..bbd32b50 100644 --- a/internal/service/health.go +++ b/internal/service/health.go @@ -3,9 +3,9 @@ package service import ( "context" - "github.com/go-redis/redis/v8" "github.com/nats-io/nats.go" "github.com/pkg/errors" + "github.com/redis/go-redis/v9" "github.com/uptrace/bun" ) diff --git a/internal/service/report.go b/internal/service/report.go index 2453b2e9..db8cd5c5 100644 --- a/internal/service/report.go +++ b/internal/service/report.go @@ -6,11 +6,11 @@ import ( "exusiai.dev/gommon/constant" "github.com/dchest/uniuri" - "github.com/go-redis/redis/v8" "github.com/goccy/go-json" "github.com/gofiber/fiber/v2" "github.com/nats-io/nats.go" "github.com/pkg/errors" + "github.com/redis/go-redis/v9" "github.com/uptrace/bun" "exusiai.dev/backend-next/internal/model/types" diff --git a/internal/workers/reportwkr/reportwkr.go b/internal/workers/reportwkr/reportwkr.go index aa7829bc..80aff7af 100644 --- a/internal/workers/reportwkr/reportwkr.go +++ b/internal/workers/reportwkr/reportwkr.go @@ -7,10 +7,10 @@ import ( "time" "exusiai.dev/gommon/constant" - "github.com/go-redis/redis/v8" "github.com/goccy/go-json" "github.com/nats-io/nats.go" "github.com/pkg/errors" + "github.com/redis/go-redis/v9" "github.com/rs/zerolog/log" "github.com/uptrace/bun" "go.opentelemetry.io/otel" From 216e79e2214424d8c0267591eefaf74fbc435ea2 Mon Sep 17 00:00:00 2001 From: GalvinGao Date: Thu, 6 Apr 2023 14:50:46 -0400 Subject: [PATCH 6/9] test: remove wrong assertion --- test/v2_advanced_query_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/v2_advanced_query_test.go b/test/v2_advanced_query_test.go index 0eb56cf7..b004fa2f 100644 --- a/test/v2_advanced_query_test.go +++ b/test/v2_advanced_query_test.go @@ -24,8 +24,7 @@ func TestAPIV2AdvancedQuery(t *testing.T) { }) t.Run("invalid body", func(t *testing.T) { - h, j := query(`{"queries":[{"stageId":"main_01-07","itemIds":[],"server":"CN","isPersonal":false,"sourceCategory":"all","start":1556668800000,"end":1556668800000}]}`, nil) + h, _ := query(`{"queries":[{"stageId":"main_01-07","itemIds":[],"server":"CN","isPersonal":false,"sourceCategory":"all","start":1556668800000,"end":1556668800000}]}`, nil) assert.Equal(t, http.StatusBadRequest, h.StatusCode) - assert.Equal(t, "invalid time range", j.Get("error").String()) }) } From 41871cc0706e2a005ed9c7a888c8cee7a880fb91 Mon Sep 17 00:00:00 2001 From: AlvISsReimu Date: Thu, 6 Apr 2023 14:09:48 -0700 Subject: [PATCH 7/9] fix: should use latest max accumulable timerange to calculate drop matrix result from elements --- internal/repo/drop_matrix_element.go | 28 ++++++-- internal/service/drop_matrix.go | 95 ++++++++++++++++++------- internal/service/drop_matrix_element.go | 18 +++-- 3 files changed, 103 insertions(+), 38 deletions(-) diff --git a/internal/repo/drop_matrix_element.go b/internal/repo/drop_matrix_element.go index e6d65c46..54cac803 100644 --- a/internal/repo/drop_matrix_element.go +++ b/internal/repo/drop_matrix_element.go @@ -3,6 +3,7 @@ package repo import ( "context" "database/sql" + "time" "exusiai.dev/gommon/constant" "github.com/pkg/errors" @@ -63,13 +64,18 @@ func (s *DropMatrixElement) IsExistByServerAndDayNum(ctx context.Context, server return exists, nil } -func (s *DropMatrixElement) GetAllTimesForGlobalDropMatrix(ctx context.Context, server string, sourceCategory string) ([]*model.AllTimesResultForGlobalDropMatrix, error) { +func (s *DropMatrixElement) GetAllTimesForGlobalDropMatrix( + ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string, +) ([]*model.AllTimesResultForGlobalDropMatrix, error) { subq2 := s.db.NewSelect(). TableExpr("drop_matrix_elements"). Column("stage_id", "item_id", "times", "day_num"). Where("server = ?", server). Where("source_category = ?", sourceCategory). - Where("times > 0") + Where("times > 0"). + Where("stage_id IN (?)", bun.In(stageIds)). + Where("start_time >= timestamp with time zone ?", timeRange.StartTime.Format(time.RFC3339)). + Where("end_time <= timestamp with time zone ?", timeRange.EndTime.Format(time.RFC3339)) subq1 := s.db.NewSelect(). TableExpr("(?) AS subq2", subq2). @@ -90,13 +96,18 @@ func (s *DropMatrixElement) GetAllTimesForGlobalDropMatrix(ctx context.Context, return results, nil } -func (s *DropMatrixElement) GetAllQuantitiesForGlobalDropMatrix(ctx context.Context, server string, sourceCategory string) ([]*model.AllQuantitiesResultForGlobalDropMatrix, error) { +func (s *DropMatrixElement) GetAllQuantitiesForGlobalDropMatrix( + ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string, +) ([]*model.AllQuantitiesResultForGlobalDropMatrix, error) { subq1 := s.db.NewSelect(). TableExpr("drop_matrix_elements"). Column("stage_id", "item_id", "quantity"). Where("server = ?", server). Where("source_category = ?", sourceCategory). - Where("quantity > 0") + Where("quantity > 0"). + Where("stage_id IN (?)", bun.In(stageIds)). + Where("start_time >= timestamp with time zone ?", timeRange.StartTime.Format(time.RFC3339)). + Where("end_time <= timestamp with time zone ?", timeRange.EndTime.Format(time.RFC3339)) mainq := s.db.NewSelect(). TableExpr("(?) AS subq1", subq1). @@ -112,13 +123,18 @@ func (s *DropMatrixElement) GetAllQuantitiesForGlobalDropMatrix(ctx context.Cont return results, nil } -func (s *DropMatrixElement) GetAllQuantityBucketsForGlobalDropMatrix(ctx context.Context, server string, sourceCategory string) ([]*model.AllQuantityBucketsResultForGlobalDropMatrix, error) { +func (s *DropMatrixElement) GetAllQuantityBucketsForGlobalDropMatrix( + ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string, +) ([]*model.AllQuantityBucketsResultForGlobalDropMatrix, error) { subq2 := s.db.NewSelect(). TableExpr("drop_matrix_elements"). Column("stage_id", "item_id", "quantity_buckets"). Where("server = ?", server). Where("source_category = ?", sourceCategory). - Where("quantity > 0") + Where("quantity > 0"). + Where("stage_id IN (?)", bun.In(stageIds)). + Where("start_time >= timestamp with time zone ?", timeRange.StartTime.Format(time.RFC3339)). + Where("end_time <= timestamp with time zone ?", timeRange.EndTime.Format(time.RFC3339)) subq1 := s.db.NewSelect(). TableExpr("(?) AS subq2", subq2). diff --git a/internal/service/drop_matrix.go b/internal/service/drop_matrix.go index 565a8489..5cd3c4e3 100644 --- a/internal/service/drop_matrix.go +++ b/internal/service/drop_matrix.go @@ -312,40 +312,83 @@ func (s *DropMatrix) calcDropMatrix(ctx context.Context, queryCtx *model.DropRep } func (s *DropMatrix) calcGlobalDropMatrix(ctx context.Context, server string, sourceCategory string) (*model.DropMatrixQueryResult, error) { - timesResults, err := s.DropMatrixElementService.GetAllTimesForGlobalDropMatrixMapByStageIdAndItemId(ctx, server, sourceCategory) - if err != nil { - return nil, err - } - quantityResults, err := s.DropMatrixElementService.GetAllQuantitiesForGlobalDropMatrixMapByStageIdAndItemId(ctx, server, sourceCategory) - if err != nil { - return nil, err - } - quantityUniqCountResults, err := s.DropMatrixElementService.GetAllQuantityBucketsForGlobalDropMatrixMapByStageIdAndItemId(ctx, server, sourceCategory) - if err != nil { - return nil, err + finalResult := &model.DropMatrixQueryResult{ + Matrix: make([]*model.OneDropMatrixElement, 0), } + maxAccumulableTimeRanges, err := s.TimeRangeService.GetAllMaxAccumulableTimeRangesByServer(ctx, server) if err != nil { return nil, err } - finalResult := &model.DropMatrixQueryResult{ - Matrix: make([]*model.OneDropMatrixElement, 0), + // Only consider the latest (max accumulable) time range for each stage + latestMaxAccumulableTimeRanges := make(map[int]map[int]*model.TimeRange, 0) + for stageId, timeRangesMapByItemId := range maxAccumulableTimeRanges { + if _, ok := latestMaxAccumulableTimeRanges[stageId]; !ok { + latestMaxAccumulableTimeRanges[stageId] = make(map[int]*model.TimeRange, 0) + } + for itemId, timeRanges := range timeRangesMapByItemId { + latestMaxAccumulableTimeRanges[stageId][itemId] = timeRanges[len(timeRanges)-1] + } + } + + stageIdsItemIdsMapByTimeRangeStr := make(map[string]map[int][]int, 0) + for stageId, timeRangeMapByItemId := range latestMaxAccumulableTimeRanges { + for itemId, timeRange := range timeRangeMapByItemId { + timeRangeStr := timeRange.String() + if _, ok := stageIdsItemIdsMapByTimeRangeStr[timeRangeStr]; !ok { + stageIdsItemIdsMapByTimeRangeStr[timeRangeStr] = make(map[int][]int, 0) + } + if _, ok := stageIdsItemIdsMapByTimeRangeStr[timeRangeStr][stageId]; !ok { + stageIdsItemIdsMapByTimeRangeStr[timeRangeStr][stageId] = make([]int, 0) + } + stageIdsItemIdsMapByTimeRangeStr[timeRangeStr][stageId] = append(stageIdsItemIdsMapByTimeRangeStr[timeRangeStr][stageId], itemId) + } } - for stageId, subMap := range quantityResults { - for itemId, quantityResult := range subMap { - timesResult := timesResults[stageId][itemId] - quantityUniqCountResult := quantityUniqCountResults[stageId][itemId] - maxAccumulableTimeRanges := maxAccumulableTimeRanges[stageId][itemId] - oneDropMatrixElement := &model.OneDropMatrixElement{ - StageID: stageId, - ItemID: itemId, - Times: timesResult.Times, - Quantity: quantityResult.Quantity, - StdDev: util.RoundFloat64(util.CalcStdDevFromQuantityBuckets(quantityUniqCountResult.QuantityBuckets, timesResult.Times, false), constant.StdDevDigits), - TimeRange: maxAccumulableTimeRanges[0], + for timeRangeStr, stageIdsItemIdsMap := range stageIdsItemIdsMapByTimeRangeStr { + timeRange := model.TimeRangeFromString(timeRangeStr) + stageIds := make([]int, 0) + for stageId := range stageIdsItemIdsMap { + stageIds = append(stageIds, stageId) + } + + timesResults, err := s.DropMatrixElementService.GetAllTimesForGlobalDropMatrixMapByStageIdAndItemId(ctx, server, timeRange, stageIds, sourceCategory) + if err != nil { + return nil, err + } + quantityResults, err := s.DropMatrixElementService.GetAllQuantitiesForGlobalDropMatrixMapByStageIdAndItemId(ctx, server, timeRange, stageIds, sourceCategory) + if err != nil { + return nil, err + } + quantityUniqCountResults, err := s.DropMatrixElementService.GetAllQuantityBucketsForGlobalDropMatrixMapByStageIdAndItemId(ctx, server, timeRange, stageIds, sourceCategory) + if err != nil { + return nil, err + } + + for stageId, itemIds := range stageIdsItemIdsMap { + for _, itemId := range itemIds { + timesResult, foundTimesResult := timesResults[stageId][itemId] + if !foundTimesResult { + continue + } + quantityResult, foundQuantityResult := quantityResults[stageId][itemId] + if !foundQuantityResult { + continue + } + quantityUniqCountResult, foundQuantityUniqCountResult := quantityUniqCountResults[stageId][itemId] + if !foundQuantityUniqCountResult { + continue + } + oneDropMatrixElement := &model.OneDropMatrixElement{ + StageID: stageId, + ItemID: itemId, + Times: timesResult.Times, + Quantity: quantityResult.Quantity, + TimeRange: timeRange, + StdDev: util.RoundFloat64(util.CalcStdDevFromQuantityBuckets(quantityUniqCountResult.QuantityBuckets, timesResult.Times, false), constant.StdDevDigits), + } + finalResult.Matrix = append(finalResult.Matrix, oneDropMatrixElement) } - finalResult.Matrix = append(finalResult.Matrix, oneDropMatrixElement) } } return finalResult, nil diff --git a/internal/service/drop_matrix_element.go b/internal/service/drop_matrix_element.go index d958c7f3..85f4cab3 100644 --- a/internal/service/drop_matrix_element.go +++ b/internal/service/drop_matrix_element.go @@ -36,8 +36,10 @@ func (s *DropMatrixElement) IsExistByServerAndDayNum(ctx context.Context, server return s.DropMatrixElementRepo.IsExistByServerAndDayNum(ctx, server, dayNum) } -func (s *DropMatrixElement) GetAllTimesForGlobalDropMatrixMapByStageIdAndItemId(ctx context.Context, server string, sourceCategory string) (map[int]map[int]*model.AllTimesResultForGlobalDropMatrix, error) { - allTimes, err := s.DropMatrixElementRepo.GetAllTimesForGlobalDropMatrix(ctx, server, sourceCategory) +func (s *DropMatrixElement) GetAllTimesForGlobalDropMatrixMapByStageIdAndItemId( + ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string, +) (map[int]map[int]*model.AllTimesResultForGlobalDropMatrix, error) { + allTimes, err := s.DropMatrixElementRepo.GetAllTimesForGlobalDropMatrix(ctx, server, timeRange, stageIds, sourceCategory) if err != nil { return nil, err } @@ -51,8 +53,10 @@ func (s *DropMatrixElement) GetAllTimesForGlobalDropMatrixMapByStageIdAndItemId( return result, nil } -func (s *DropMatrixElement) GetAllQuantitiesForGlobalDropMatrixMapByStageIdAndItemId(ctx context.Context, server string, sourceCategory string) (map[int]map[int]*model.AllQuantitiesResultForGlobalDropMatrix, error) { - allQuantities, err := s.DropMatrixElementRepo.GetAllQuantitiesForGlobalDropMatrix(ctx, server, sourceCategory) +func (s *DropMatrixElement) GetAllQuantitiesForGlobalDropMatrixMapByStageIdAndItemId( + ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string, +) (map[int]map[int]*model.AllQuantitiesResultForGlobalDropMatrix, error) { + allQuantities, err := s.DropMatrixElementRepo.GetAllQuantitiesForGlobalDropMatrix(ctx, server, timeRange, stageIds, sourceCategory) if err != nil { return nil, err } @@ -66,8 +70,10 @@ func (s *DropMatrixElement) GetAllQuantitiesForGlobalDropMatrixMapByStageIdAndIt return result, nil } -func (s *DropMatrixElement) GetAllQuantityBucketsForGlobalDropMatrixMapByStageIdAndItemId(ctx context.Context, server string, sourceCategory string) (map[int]map[int]*model.AllQuantityBucketsResultForGlobalDropMatrix, error) { - allQuantityBuckets, err := s.DropMatrixElementRepo.GetAllQuantityBucketsForGlobalDropMatrix(ctx, server, sourceCategory) +func (s *DropMatrixElement) GetAllQuantityBucketsForGlobalDropMatrixMapByStageIdAndItemId( + ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string, +) (map[int]map[int]*model.AllQuantityBucketsResultForGlobalDropMatrix, error) { + allQuantityBuckets, err := s.DropMatrixElementRepo.GetAllQuantityBucketsForGlobalDropMatrix(ctx, server, timeRange, stageIds, sourceCategory) if err != nil { return nil, err } From ca50d71165a17f53379e144c3b45e903e685d789 Mon Sep 17 00:00:00 2001 From: AlvISsReimu Date: Thu, 6 Apr 2023 15:29:27 -0700 Subject: [PATCH 8/9] fix: should use latest timerange to calculate pattern matrix result from elements --- internal/repo/pattern_matrix_element.go | 19 ++++++-- internal/service/pattern_matrix.go | 53 +++++++++++++--------- internal/service/pattern_matrix_element.go | 12 +++-- 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/internal/repo/pattern_matrix_element.go b/internal/repo/pattern_matrix_element.go index e331d7bc..48631821 100644 --- a/internal/repo/pattern_matrix_element.go +++ b/internal/repo/pattern_matrix_element.go @@ -2,6 +2,7 @@ package repo import ( "context" + "time" "github.com/uptrace/bun" @@ -37,13 +38,18 @@ func (s *PatternMatrixElement) IsExistByServerAndDayNum(ctx context.Context, ser return exists, nil } -func (s *PatternMatrixElement) GetAllTimesForGlobalPatternMatrix(ctx context.Context, server string, sourceCategory string) ([]*model.AllTimesResultForGlobalPatternMatrix, error) { +func (s *PatternMatrixElement) GetAllTimesForGlobalPatternMatrix( + ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string, +) ([]*model.AllTimesResultForGlobalPatternMatrix, error) { subq2 := s.db.NewSelect(). TableExpr("pattern_matrix_elements"). Column("stage_id", "times", "day_num"). Where("server = ?", server). Where("source_category = ?", sourceCategory). - Where("times > 0") + Where("times > 0"). + Where("stage_id IN (?)", bun.In(stageIds)). + Where("start_time >= timestamp with time zone ?", timeRange.StartTime.Format(time.RFC3339)). + Where("end_time <= timestamp with time zone ?", timeRange.EndTime.Format(time.RFC3339)) subq1 := s.db.NewSelect(). TableExpr("(?) AS subq2", subq2). @@ -65,13 +71,18 @@ func (s *PatternMatrixElement) GetAllTimesForGlobalPatternMatrix(ctx context.Con return results, nil } -func (s *PatternMatrixElement) GetAllQuantitiesForGlobalPatternMatrix(ctx context.Context, server string, sourceCategory string) ([]*model.AllQuantitiesResultForGlobalPatternMatrix, error) { +func (s *PatternMatrixElement) GetAllQuantitiesForGlobalPatternMatrix( + ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string, +) ([]*model.AllQuantitiesResultForGlobalPatternMatrix, error) { subq1 := s.db.NewSelect(). TableExpr("pattern_matrix_elements"). Column("stage_id", "pattern_id", "quantity"). Where("server = ?", server). Where("source_category = ?", sourceCategory). - Where("quantity > 0") + Where("quantity > 0"). + Where("stage_id IN (?)", bun.In(stageIds)). + Where("start_time >= timestamp with time zone ?", timeRange.StartTime.Format(time.RFC3339)). + Where("end_time <= timestamp with time zone ?", timeRange.EndTime.Format(time.RFC3339)) mainq := s.db.NewSelect(). TableExpr("(?) AS subq1", subq1). diff --git a/internal/service/pattern_matrix.go b/internal/service/pattern_matrix.go index 41078e69..d8fbd25b 100644 --- a/internal/service/pattern_matrix.go +++ b/internal/service/pattern_matrix.go @@ -213,34 +213,45 @@ func (s *PatternMatrix) calcPatternMatrixByGivenDate( } func (s *PatternMatrix) calcGlobalPatternMatrix(ctx context.Context, server string, sourceCategory string) (*model.PatternMatrixQueryResult, error) { - timesResults, err := s.PatternMatrixElementService.GetAllTimesForGlobalPatternMatrixMapByStageId(ctx, server, sourceCategory) - if err != nil { - return nil, err - } - quantityResults, err := s.PatternMatrixElementService.GetAllQuantitiesForGlobalPatternMatrixMapByStageIdAndItemId(ctx, server, sourceCategory) - if err != nil { - return nil, err + finalResult := &model.PatternMatrixQueryResult{ + PatternMatrix: make([]*model.OnePatternMatrixElement, 0), } + latestTimeRanges, err := s.TimeRangeService.GetLatestTimeRangesByServer(ctx, server) if err != nil { return nil, err } - - finalResult := &model.PatternMatrixQueryResult{ - PatternMatrix: make([]*model.OnePatternMatrixElement, 0), + stageIdsMapByTimeRangeStr := make(map[string][]int, 0) + for stageId, timeRange := range latestTimeRanges { + timeRangeStr := timeRange.String() + if _, ok := stageIdsMapByTimeRangeStr[timeRangeStr]; !ok { + stageIdsMapByTimeRangeStr[timeRangeStr] = make([]int, 0) + } + stageIdsMapByTimeRangeStr[timeRangeStr] = append(stageIdsMapByTimeRangeStr[timeRangeStr], stageId) } - for stageId, subMap := range quantityResults { - timesResult := timesResults[stageId] - latestTimeRange := latestTimeRanges[stageId] - for patternId, quantityResult := range subMap { - onePatternMatrixElement := &model.OnePatternMatrixElement{ - StageID: stageId, - PatternID: patternId, - TimeRange: latestTimeRange, - Times: timesResult.Times, - Quantity: quantityResult.Quantity, + for timeRangeStr, stageIds := range stageIdsMapByTimeRangeStr { + timeRange := model.TimeRangeFromString(timeRangeStr) + + timesResults, err := s.PatternMatrixElementService.GetAllTimesForGlobalPatternMatrixMapByStageId(ctx, server, timeRange, stageIds, sourceCategory) + if err != nil { + return nil, err + } + quantityResults, err := s.PatternMatrixElementService.GetAllQuantitiesForGlobalPatternMatrixMapByStageIdAndPatternId(ctx, server, timeRange, stageIds, sourceCategory) + if err != nil { + return nil, err + } + for _, stageId := range stageIds { + timesResult := timesResults[stageId] + for patternId, quantityResult := range quantityResults[stageId] { + onePatternMatrixElement := &model.OnePatternMatrixElement{ + StageID: stageId, + PatternID: patternId, + TimeRange: timeRange, + Times: timesResult.Times, + Quantity: quantityResult.Quantity, + } + finalResult.PatternMatrix = append(finalResult.PatternMatrix, onePatternMatrixElement) } - finalResult.PatternMatrix = append(finalResult.PatternMatrix, onePatternMatrixElement) } } return finalResult, nil diff --git a/internal/service/pattern_matrix_element.go b/internal/service/pattern_matrix_element.go index d8ac4c8b..6f2a7d3f 100644 --- a/internal/service/pattern_matrix_element.go +++ b/internal/service/pattern_matrix_element.go @@ -29,8 +29,10 @@ func (s *PatternMatrixElement) IsExistByServerAndDayNum(ctx context.Context, ser return s.PatternMatrixElementRepo.IsExistByServerAndDayNum(ctx, server, dayNum) } -func (s *PatternMatrixElement) GetAllTimesForGlobalPatternMatrixMapByStageId(ctx context.Context, server string, sourceCategory string) (map[int]*model.AllTimesResultForGlobalPatternMatrix, error) { - allTimes, err := s.PatternMatrixElementRepo.GetAllTimesForGlobalPatternMatrix(ctx, server, sourceCategory) +func (s *PatternMatrixElement) GetAllTimesForGlobalPatternMatrixMapByStageId( + ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string, +) (map[int]*model.AllTimesResultForGlobalPatternMatrix, error) { + allTimes, err := s.PatternMatrixElementRepo.GetAllTimesForGlobalPatternMatrix(ctx, server, timeRange, stageIds, sourceCategory) if err != nil { return nil, err } @@ -41,8 +43,10 @@ func (s *PatternMatrixElement) GetAllTimesForGlobalPatternMatrixMapByStageId(ctx return result, nil } -func (s *PatternMatrixElement) GetAllQuantitiesForGlobalPatternMatrixMapByStageIdAndItemId(ctx context.Context, server string, sourceCategory string) (map[int]map[int]*model.AllQuantitiesResultForGlobalPatternMatrix, error) { - allQuantities, err := s.PatternMatrixElementRepo.GetAllQuantitiesForGlobalPatternMatrix(ctx, server, sourceCategory) +func (s *PatternMatrixElement) GetAllQuantitiesForGlobalPatternMatrixMapByStageIdAndPatternId( + ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string, +) (map[int]map[int]*model.AllQuantitiesResultForGlobalPatternMatrix, error) { + allQuantities, err := s.PatternMatrixElementRepo.GetAllQuantitiesForGlobalPatternMatrix(ctx, server, timeRange, stageIds, sourceCategory) if err != nil { return nil, err } From 143d52a904d281c4b7658a332a1db7554e7f917e Mon Sep 17 00:00:00 2001 From: AlvISsReimu Date: Thu, 6 Apr 2023 15:40:33 -0700 Subject: [PATCH 9/9] fix: during shim, if end time is after now, set it to be null, so that the frontend will show it as till now --- internal/service/drop_matrix.go | 11 +++++++---- internal/service/pattern_matrix.go | 12 ++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/internal/service/drop_matrix.go b/internal/service/drop_matrix.go index 5cd3c4e3..b8d69bd1 100644 --- a/internal/service/drop_matrix.go +++ b/internal/service/drop_matrix.go @@ -915,7 +915,13 @@ func (s *DropMatrix) applyShimForDropMatrixQuery(ctx context.Context, server str } } - endTime := null.NewInt(el.TimeRange.EndTime.UnixMilli(), true) + // if end time is after now, set it to be null, so that the frontend will show it as "till now" + var endTime null.Int + if el.TimeRange.EndTime.After(time.Now()) { + endTime = null.NewInt(0, false) + } else { + endTime = null.NewInt(el.TimeRange.EndTime.UnixMilli(), true) + } oneDropMatrixElement := modelv2.OneDropMatrixElement{ StageID: stage.ArkStageID, ItemID: item.ArkItemID, @@ -925,9 +931,6 @@ func (s *DropMatrix) applyShimForDropMatrixQuery(ctx context.Context, server str StartTime: el.TimeRange.StartTime.UnixMilli(), EndTime: endTime, } - if oneDropMatrixElement.EndTime.Int64 == constant.FakeEndTimeMilli { - oneDropMatrixElement.EndTime = null.NewInt(0, false) - } results.Matrix = append(results.Matrix, &oneDropMatrixElement) } return results, nil diff --git a/internal/service/pattern_matrix.go b/internal/service/pattern_matrix.go index d8fbd25b..4a9fbb59 100644 --- a/internal/service/pattern_matrix.go +++ b/internal/service/pattern_matrix.go @@ -494,7 +494,6 @@ func (s *PatternMatrix) applyShimForPatternMatrixQuery(ctx context.Context, quer for _, el := range group.Group { oneDropPattern := el.(*model.OnePatternMatrixElement) stage := stagesMapById[oneDropPattern.StageID] - endTime := null.NewInt(oneDropPattern.TimeRange.EndTime.UnixMilli(), true) dropPatternElements, err := s.DropPatternElementService.GetDropPatternElementsByPatternId(ctx, patternId) if err != nil { return nil, err @@ -516,6 +515,14 @@ func (s *PatternMatrix) applyShimForPatternMatrixQuery(ctx context.Context, quer Quantity: dropPatternElement.Quantity, }) } + + // if end time is after now, set it to be null, so that the frontend will show it as "till now" + var endTime null.Int + if oneDropPattern.TimeRange.EndTime.After(time.Now()) { + endTime = null.NewInt(0, false) + } else { + endTime = null.NewInt(oneDropPattern.TimeRange.EndTime.UnixMilli(), true) + } onePatternMatrixElement := modelv2.OnePatternMatrixElement{ StageID: stage.ArkStageID, Times: oneDropPattern.Times, @@ -524,9 +531,6 @@ func (s *PatternMatrix) applyShimForPatternMatrixQuery(ctx context.Context, quer EndTime: endTime, Pattern: &pattern, } - if onePatternMatrixElement.EndTime.Int64 == constant.FakeEndTimeMilli { - onePatternMatrixElement.EndTime = null.NewInt(0, false) - } results.PatternMatrix = append(results.PatternMatrix, &onePatternMatrixElement) } }