Skip to content

Commit

Permalink
Merge pull request #432 from penguin-statistics/dev
Browse files Browse the repository at this point in the history 
Release v3.11.8
  • Loading branch information
@AlvISsReimu
AlvISsReimu authored Apr 6, 2023
2 parents 0a4acd4 + 184ea68 commit 7378b62
Show file tree
Hide file tree
Showing 7 changed files with 174 additions and 77 deletions.
28 changes: 22 additions & 6 deletions internal/repo/drop_matrix_element.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package repo
import (
"context"
"database/sql"
"time"

"exusiai.dev/gommon/constant"
"github.com/pkg/errors"
Expand Down Expand Up @@ -63,13 +64,18 @@ func (s *DropMatrixElement) IsExistByServerAndDayNum(ctx context.Context, server
return exists, nil
}

func (s *DropMatrixElement) GetAllTimesForGlobalDropMatrix(ctx context.Context, server string, sourceCategory string) ([]*model.AllTimesResultForGlobalDropMatrix, error) {
func (s *DropMatrixElement) GetAllTimesForGlobalDropMatrix(
ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string,
) ([]*model.AllTimesResultForGlobalDropMatrix, error) {
subq2 := s.db.NewSelect().
TableExpr("drop_matrix_elements").
Column("stage_id", "item_id", "times", "day_num").
Where("server = ?", server).
Where("source_category = ?", sourceCategory).
Where("times > 0")
Where("times > 0").
Where("stage_id IN (?)", bun.In(stageIds)).
Where("start_time >= timestamp with time zone ?", timeRange.StartTime.Format(time.RFC3339)).
Where("end_time <= timestamp with time zone ?", timeRange.EndTime.Format(time.RFC3339))

subq1 := s.db.NewSelect().
TableExpr("(?) AS subq2", subq2).
Expand All @@ -90,13 +96,18 @@ func (s *DropMatrixElement) GetAllTimesForGlobalDropMatrix(ctx context.Context,
return results, nil
}

func (s *DropMatrixElement) GetAllQuantitiesForGlobalDropMatrix(ctx context.Context, server string, sourceCategory string) ([]*model.AllQuantitiesResultForGlobalDropMatrix, error) {
func (s *DropMatrixElement) GetAllQuantitiesForGlobalDropMatrix(
ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string,
) ([]*model.AllQuantitiesResultForGlobalDropMatrix, error) {
subq1 := s.db.NewSelect().
TableExpr("drop_matrix_elements").
Column("stage_id", "item_id", "quantity").
Where("server = ?", server).
Where("source_category = ?", sourceCategory).
Where("quantity > 0")
Where("quantity > 0").
Where("stage_id IN (?)", bun.In(stageIds)).
Where("start_time >= timestamp with time zone ?", timeRange.StartTime.Format(time.RFC3339)).
Where("end_time <= timestamp with time zone ?", timeRange.EndTime.Format(time.RFC3339))

mainq := s.db.NewSelect().
TableExpr("(?) AS subq1", subq1).
Expand All @@ -112,13 +123,18 @@ func (s *DropMatrixElement) GetAllQuantitiesForGlobalDropMatrix(ctx context.Cont
return results, nil
}

func (s *DropMatrixElement) GetAllQuantityBucketsForGlobalDropMatrix(ctx context.Context, server string, sourceCategory string) ([]*model.AllQuantityBucketsResultForGlobalDropMatrix, error) {
func (s *DropMatrixElement) GetAllQuantityBucketsForGlobalDropMatrix(
ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string,
) ([]*model.AllQuantityBucketsResultForGlobalDropMatrix, error) {
subq2 := s.db.NewSelect().
TableExpr("drop_matrix_elements").
Column("stage_id", "item_id", "quantity_buckets").
Where("server = ?", server).
Where("source_category = ?", sourceCategory).
Where("quantity > 0")
Where("quantity > 0").
Where("stage_id IN (?)", bun.In(stageIds)).
Where("start_time >= timestamp with time zone ?", timeRange.StartTime.Format(time.RFC3339)).
Where("end_time <= timestamp with time zone ?", timeRange.EndTime.Format(time.RFC3339))

subq1 := s.db.NewSelect().
TableExpr("(?) AS subq2", subq2).
Expand Down
19 changes: 15 additions & 4 deletions internal/repo/pattern_matrix_element.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package repo

import (
"context"
"time"

"github.com/uptrace/bun"

Expand Down Expand Up @@ -37,13 +38,18 @@ func (s *PatternMatrixElement) IsExistByServerAndDayNum(ctx context.Context, ser
return exists, nil
}

func (s *PatternMatrixElement) GetAllTimesForGlobalPatternMatrix(ctx context.Context, server string, sourceCategory string) ([]*model.AllTimesResultForGlobalPatternMatrix, error) {
func (s *PatternMatrixElement) GetAllTimesForGlobalPatternMatrix(
ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string,
) ([]*model.AllTimesResultForGlobalPatternMatrix, error) {
subq2 := s.db.NewSelect().
TableExpr("pattern_matrix_elements").
Column("stage_id", "times", "day_num").
Where("server = ?", server).
Where("source_category = ?", sourceCategory).
Where("times > 0")
Where("times > 0").
Where("stage_id IN (?)", bun.In(stageIds)).
Where("start_time >= timestamp with time zone ?", timeRange.StartTime.Format(time.RFC3339)).
Where("end_time <= timestamp with time zone ?", timeRange.EndTime.Format(time.RFC3339))

subq1 := s.db.NewSelect().
TableExpr("(?) AS subq2", subq2).
Expand All @@ -65,13 +71,18 @@ func (s *PatternMatrixElement) GetAllTimesForGlobalPatternMatrix(ctx context.Con
return results, nil
}

func (s *PatternMatrixElement) GetAllQuantitiesForGlobalPatternMatrix(ctx context.Context, server string, sourceCategory string) ([]*model.AllQuantitiesResultForGlobalPatternMatrix, error) {
func (s *PatternMatrixElement) GetAllQuantitiesForGlobalPatternMatrix(
ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string,
) ([]*model.AllQuantitiesResultForGlobalPatternMatrix, error) {
subq1 := s.db.NewSelect().
TableExpr("pattern_matrix_elements").
Column("stage_id", "pattern_id", "quantity").
Where("server = ?", server).
Where("source_category = ?", sourceCategory).
Where("quantity > 0")
Where("quantity > 0").
Where("stage_id IN (?)", bun.In(stageIds)).
Where("start_time >= timestamp with time zone ?", timeRange.StartTime.Format(time.RFC3339)).
Where("end_time <= timestamp with time zone ?", timeRange.EndTime.Format(time.RFC3339))

mainq := s.db.NewSelect().
TableExpr("(?) AS subq1", subq1).
Expand Down
106 changes: 76 additions & 30 deletions internal/service/drop_matrix.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,40 +312,83 @@ func (s *DropMatrix) calcDropMatrix(ctx context.Context, queryCtx *model.DropRep
}

func (s *DropMatrix) calcGlobalDropMatrix(ctx context.Context, server string, sourceCategory string) (*model.DropMatrixQueryResult, error) {
timesResults, err := s.DropMatrixElementService.GetAllTimesForGlobalDropMatrixMapByStageIdAndItemId(ctx, server, sourceCategory)
if err != nil {
return nil, err
}
quantityResults, err := s.DropMatrixElementService.GetAllQuantitiesForGlobalDropMatrixMapByStageIdAndItemId(ctx, server, sourceCategory)
if err != nil {
return nil, err
}
quantityUniqCountResults, err := s.DropMatrixElementService.GetAllQuantityBucketsForGlobalDropMatrixMapByStageIdAndItemId(ctx, server, sourceCategory)
if err != nil {
return nil, err
finalResult := &model.DropMatrixQueryResult{
Matrix: make([]*model.OneDropMatrixElement, 0),
}

maxAccumulableTimeRanges, err := s.TimeRangeService.GetAllMaxAccumulableTimeRangesByServer(ctx, server)
if err != nil {
return nil, err
}

finalResult := &model.DropMatrixQueryResult{
Matrix: make([]*model.OneDropMatrixElement, 0),
// Only consider the latest (max accumulable) time range for each stage
latestMaxAccumulableTimeRanges := make(map[int]map[int]*model.TimeRange, 0)
for stageId, timeRangesMapByItemId := range maxAccumulableTimeRanges {
if _, ok := latestMaxAccumulableTimeRanges[stageId]; !ok {
latestMaxAccumulableTimeRanges[stageId] = make(map[int]*model.TimeRange, 0)
}
for itemId, timeRanges := range timeRangesMapByItemId {
latestMaxAccumulableTimeRanges[stageId][itemId] = timeRanges[len(timeRanges)-1]
}
}
for stageId, subMap := range quantityResults {
for itemId, quantityResult := range subMap {
timesResult := timesResults[stageId][itemId]
quantityUniqCountResult := quantityUniqCountResults[stageId][itemId]
maxAccumulableTimeRanges := maxAccumulableTimeRanges[stageId][itemId]
oneDropMatrixElement := &model.OneDropMatrixElement{
StageID: stageId,
ItemID: itemId,
Times: timesResult.Times,
Quantity: quantityResult.Quantity,
StdDev: util.RoundFloat64(util.CalcStdDevFromQuantityBuckets(quantityUniqCountResult.QuantityBuckets, timesResult.Times, false), constant.StdDevDigits),
TimeRange: maxAccumulableTimeRanges[0],

stageIdsItemIdsMapByTimeRangeStr := make(map[string]map[int][]int, 0)
for stageId, timeRangeMapByItemId := range latestMaxAccumulableTimeRanges {
for itemId, timeRange := range timeRangeMapByItemId {
timeRangeStr := timeRange.String()
if _, ok := stageIdsItemIdsMapByTimeRangeStr[timeRangeStr]; !ok {
stageIdsItemIdsMapByTimeRangeStr[timeRangeStr] = make(map[int][]int, 0)
}
if _, ok := stageIdsItemIdsMapByTimeRangeStr[timeRangeStr][stageId]; !ok {
stageIdsItemIdsMapByTimeRangeStr[timeRangeStr][stageId] = make([]int, 0)
}
stageIdsItemIdsMapByTimeRangeStr[timeRangeStr][stageId] = append(stageIdsItemIdsMapByTimeRangeStr[timeRangeStr][stageId], itemId)
}
}
for timeRangeStr, stageIdsItemIdsMap := range stageIdsItemIdsMapByTimeRangeStr {
timeRange := model.TimeRangeFromString(timeRangeStr)
stageIds := make([]int, 0)
for stageId := range stageIdsItemIdsMap {
stageIds = append(stageIds, stageId)
}

timesResults, err := s.DropMatrixElementService.GetAllTimesForGlobalDropMatrixMapByStageIdAndItemId(ctx, server, timeRange, stageIds, sourceCategory)
if err != nil {
return nil, err
}
quantityResults, err := s.DropMatrixElementService.GetAllQuantitiesForGlobalDropMatrixMapByStageIdAndItemId(ctx, server, timeRange, stageIds, sourceCategory)
if err != nil {
return nil, err
}
quantityUniqCountResults, err := s.DropMatrixElementService.GetAllQuantityBucketsForGlobalDropMatrixMapByStageIdAndItemId(ctx, server, timeRange, stageIds, sourceCategory)
if err != nil {
return nil, err
}

for stageId, itemIds := range stageIdsItemIdsMap {
for _, itemId := range itemIds {
timesResult, foundTimesResult := timesResults[stageId][itemId]
if !foundTimesResult {
continue
}
quantityResult, foundQuantityResult := quantityResults[stageId][itemId]
if !foundQuantityResult {
continue
}
quantityUniqCountResult, foundQuantityUniqCountResult := quantityUniqCountResults[stageId][itemId]
if !foundQuantityUniqCountResult {
continue
}
oneDropMatrixElement := &model.OneDropMatrixElement{
StageID: stageId,
ItemID: itemId,
Times: timesResult.Times,
Quantity: quantityResult.Quantity,
TimeRange: timeRange,
StdDev: util.RoundFloat64(util.CalcStdDevFromQuantityBuckets(quantityUniqCountResult.QuantityBuckets, timesResult.Times, false), constant.StdDevDigits),
}
finalResult.Matrix = append(finalResult.Matrix, oneDropMatrixElement)
}
finalResult.Matrix = append(finalResult.Matrix, oneDropMatrixElement)
}
}
return finalResult, nil
Expand Down Expand Up @@ -872,7 +915,13 @@ func (s *DropMatrix) applyShimForDropMatrixQuery(ctx context.Context, server str
}
}

endTime := null.NewInt(el.TimeRange.EndTime.UnixMilli(), true)
// if end time is after now, set it to be null, so that the frontend will show it as "till now"
var endTime null.Int
if el.TimeRange.EndTime.After(time.Now()) {
endTime = null.NewInt(0, false)
} else {
endTime = null.NewInt(el.TimeRange.EndTime.UnixMilli(), true)
}
oneDropMatrixElement := modelv2.OneDropMatrixElement{
StageID: stage.ArkStageID,
ItemID: item.ArkItemID,
Expand All @@ -882,9 +931,6 @@ func (s *DropMatrix) applyShimForDropMatrixQuery(ctx context.Context, server str
StartTime: el.TimeRange.StartTime.UnixMilli(),
EndTime: endTime,
}
if oneDropMatrixElement.EndTime.Int64 == constant.FakeEndTimeMilli {
oneDropMatrixElement.EndTime = null.NewInt(0, false)
}
results.Matrix = append(results.Matrix, &oneDropMatrixElement)
}
return results, nil
Expand Down
18 changes: 12 additions & 6 deletions internal/service/drop_matrix_element.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,10 @@ func (s *DropMatrixElement) IsExistByServerAndDayNum(ctx context.Context, server
return s.DropMatrixElementRepo.IsExistByServerAndDayNum(ctx, server, dayNum)
}

func (s *DropMatrixElement) GetAllTimesForGlobalDropMatrixMapByStageIdAndItemId(ctx context.Context, server string, sourceCategory string) (map[int]map[int]*model.AllTimesResultForGlobalDropMatrix, error) {
allTimes, err := s.DropMatrixElementRepo.GetAllTimesForGlobalDropMatrix(ctx, server, sourceCategory)
func (s *DropMatrixElement) GetAllTimesForGlobalDropMatrixMapByStageIdAndItemId(
ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string,
) (map[int]map[int]*model.AllTimesResultForGlobalDropMatrix, error) {
allTimes, err := s.DropMatrixElementRepo.GetAllTimesForGlobalDropMatrix(ctx, server, timeRange, stageIds, sourceCategory)
if err != nil {
return nil, err
}
Expand All @@ -51,8 +53,10 @@ func (s *DropMatrixElement) GetAllTimesForGlobalDropMatrixMapByStageIdAndItemId(
return result, nil
}

func (s *DropMatrixElement) GetAllQuantitiesForGlobalDropMatrixMapByStageIdAndItemId(ctx context.Context, server string, sourceCategory string) (map[int]map[int]*model.AllQuantitiesResultForGlobalDropMatrix, error) {
allQuantities, err := s.DropMatrixElementRepo.GetAllQuantitiesForGlobalDropMatrix(ctx, server, sourceCategory)
func (s *DropMatrixElement) GetAllQuantitiesForGlobalDropMatrixMapByStageIdAndItemId(
ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string,
) (map[int]map[int]*model.AllQuantitiesResultForGlobalDropMatrix, error) {
allQuantities, err := s.DropMatrixElementRepo.GetAllQuantitiesForGlobalDropMatrix(ctx, server, timeRange, stageIds, sourceCategory)
if err != nil {
return nil, err
}
Expand All @@ -66,8 +70,10 @@ func (s *DropMatrixElement) GetAllQuantitiesForGlobalDropMatrixMapByStageIdAndIt
return result, nil
}

func (s *DropMatrixElement) GetAllQuantityBucketsForGlobalDropMatrixMapByStageIdAndItemId(ctx context.Context, server string, sourceCategory string) (map[int]map[int]*model.AllQuantityBucketsResultForGlobalDropMatrix, error) {
allQuantityBuckets, err := s.DropMatrixElementRepo.GetAllQuantityBucketsForGlobalDropMatrix(ctx, server, sourceCategory)
func (s *DropMatrixElement) GetAllQuantityBucketsForGlobalDropMatrixMapByStageIdAndItemId(
ctx context.Context, server string, timeRange *model.TimeRange, stageIds []int, sourceCategory string,
) (map[int]map[int]*model.AllQuantityBucketsResultForGlobalDropMatrix, error) {
allQuantityBuckets, err := s.DropMatrixElementRepo.GetAllQuantityBucketsForGlobalDropMatrix(ctx, server, timeRange, stageIds, sourceCategory)
if err != nil {
return nil, err
}
Expand Down
65 changes: 40 additions & 25 deletions internal/service/pattern_matrix.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,34 +213,45 @@ func (s *PatternMatrix) calcPatternMatrixByGivenDate(
}

func (s *PatternMatrix) calcGlobalPatternMatrix(ctx context.Context, server string, sourceCategory string) (*model.PatternMatrixQueryResult, error) {
timesResults, err := s.PatternMatrixElementService.GetAllTimesForGlobalPatternMatrixMapByStageId(ctx, server, sourceCategory)
if err != nil {
return nil, err
}
quantityResults, err := s.PatternMatrixElementService.GetAllQuantitiesForGlobalPatternMatrixMapByStageIdAndItemId(ctx, server, sourceCategory)
if err != nil {
return nil, err
finalResult := &model.PatternMatrixQueryResult{
PatternMatrix: make([]*model.OnePatternMatrixElement, 0),
}

latestTimeRanges, err := s.TimeRangeService.GetLatestTimeRangesByServer(ctx, server)
if err != nil {
return nil, err
}

finalResult := &model.PatternMatrixQueryResult{
PatternMatrix: make([]*model.OnePatternMatrixElement, 0),
stageIdsMapByTimeRangeStr := make(map[string][]int, 0)
for stageId, timeRange := range latestTimeRanges {
timeRangeStr := timeRange.String()
if _, ok := stageIdsMapByTimeRangeStr[timeRangeStr]; !ok {
stageIdsMapByTimeRangeStr[timeRangeStr] = make([]int, 0)
}
stageIdsMapByTimeRangeStr[timeRangeStr] = append(stageIdsMapByTimeRangeStr[timeRangeStr], stageId)
}
for stageId, subMap := range quantityResults {
timesResult := timesResults[stageId]
latestTimeRange := latestTimeRanges[stageId]
for patternId, quantityResult := range subMap {
onePatternMatrixElement := &model.OnePatternMatrixElement{
StageID: stageId,
PatternID: patternId,
TimeRange: latestTimeRange,
Times: timesResult.Times,
Quantity: quantityResult.Quantity,
for timeRangeStr, stageIds := range stageIdsMapByTimeRangeStr {
timeRange := model.TimeRangeFromString(timeRangeStr)

timesResults, err := s.PatternMatrixElementService.GetAllTimesForGlobalPatternMatrixMapByStageId(ctx, server, timeRange, stageIds, sourceCategory)
if err != nil {
return nil, err
}
quantityResults, err := s.PatternMatrixElementService.GetAllQuantitiesForGlobalPatternMatrixMapByStageIdAndPatternId(ctx, server, timeRange, stageIds, sourceCategory)
if err != nil {
return nil, err
}
for _, stageId := range stageIds {
timesResult := timesResults[stageId]
for patternId, quantityResult := range quantityResults[stageId] {
onePatternMatrixElement := &model.OnePatternMatrixElement{
StageID: stageId,
PatternID: patternId,
TimeRange: timeRange,
Times: timesResult.Times,
Quantity: quantityResult.Quantity,
}
finalResult.PatternMatrix = append(finalResult.PatternMatrix, onePatternMatrixElement)
}
finalResult.PatternMatrix = append(finalResult.PatternMatrix, onePatternMatrixElement)
}
}
return finalResult, nil
Expand Down Expand Up @@ -483,7 +494,6 @@ func (s *PatternMatrix) applyShimForPatternMatrixQuery(ctx context.Context, quer
for _, el := range group.Group {
oneDropPattern := el.(*model.OnePatternMatrixElement)
stage := stagesMapById[oneDropPattern.StageID]
endTime := null.NewInt(oneDropPattern.TimeRange.EndTime.UnixMilli(), true)
dropPatternElements, err := s.DropPatternElementService.GetDropPatternElementsByPatternId(ctx, patternId)
if err != nil {
return nil, err
Expand All @@ -505,6 +515,14 @@ func (s *PatternMatrix) applyShimForPatternMatrixQuery(ctx context.Context, quer
Quantity: dropPatternElement.Quantity,
})
}

// if end time is after now, set it to be null, so that the frontend will show it as "till now"
var endTime null.Int
if oneDropPattern.TimeRange.EndTime.After(time.Now()) {
endTime = null.NewInt(0, false)
} else {
endTime = null.NewInt(oneDropPattern.TimeRange.EndTime.UnixMilli(), true)
}
onePatternMatrixElement := modelv2.OnePatternMatrixElement{
StageID: stage.ArkStageID,
Times: oneDropPattern.Times,
Expand All @@ -513,9 +531,6 @@ func (s *PatternMatrix) applyShimForPatternMatrixQuery(ctx context.Context, quer
EndTime: endTime,
Pattern: &pattern,
}
if onePatternMatrixElement.EndTime.Int64 == constant.FakeEndTimeMilli {
onePatternMatrixElement.EndTime = null.NewInt(0, false)
}
results.PatternMatrix = append(results.PatternMatrix, &onePatternMatrixElement)
}
}
Expand Down
Loading

0 comments on commit 7378b62

Please sign in to comment.