fix: optimize sql of test dashboard to solve the problem of slow sql

.erda/migrations/qa/20211201-auto-test-exec-history-index.sql

@@ -0,0 +1,3 @@
+ALTER TABLE `dice_autotest_exec_history` ADD INDEX `idx_step_id` (`step_id`);
+ALTER TABLE `dice_autotest_exec_history` ADD INDEX `idx_scene_id` (`scene_id`);
+ALTER TABLE `dice_autotest_exec_history` ADD INDEX `idx_project_id_iteration_id_type_execute_time` (`project_id`, `iteration_id`, `type`, `execute_time`);

apistructs/autotest_exec_history.go

@@ -51,10 +51,12 @@ type AutoTestExecHistoryResp struct {
 type ExecHistorySceneAvgCostTime struct {
 	SceneID uint64  `json:"sceneID" gorm:"scene_id"`
 	Avg     float64 `json:"avg" gorm:"avg"`
+	Name    string  `json:"name" gorm:"name"`
 }
 
 type ExecHistorySceneStatusCount struct {
 	SceneID      uint64  `json:"sceneID" gorm:"scene_id"`
+	Name         string  `json:"name" gorm:"name"`
 	SuccessCount uint64  `json:"successCount" gorm:"success_count"`
 	FailCount    uint64  `json:"failCount" gorm:"fail_count"`
 	FailRate     float64 `json:"failRate"`
@@ -64,18 +66,21 @@ type ExecHistorySceneApiStatusCount struct {
 	SceneID      uint64  `json:"sceneID" gorm:"scene_id"`
 	SuccessCount uint64  `json:"successCount" gorm:"success_count"`
 	TotalCount   uint64  `json:"failCount" gorm:"total_count"`
+	Name         string  `json:"name" gorm:"name"`
 	PassRate     float64 `json:"passRate"`
 }
 
 type ExecHistoryApiAvgCostTime struct {
 	StepID uint64  `json:"stepID" gorm:"step_id"`
 	Avg    float64 `json:"avg" gorm:"avg"`
+	Name   string  `json:"name" gorm:"name"`
 }
 
 type ExecHistoryApiStatusCount struct {
 	StepID       uint64  `json:"stepID" gorm:"step_id"`
 	SuccessCount uint64  `json:"successCount" gorm:"success_count"`
 	FailCount    uint64  `json:"failCount" gorm:"fail_count"`
+	Name         string  `json:"name" gorm:"name"`
 	FailRate     float64 `json:"failRate"`
 }
 

apistructs/autotest_space.go

@@ -234,3 +234,11 @@ type AutoTestSpaceStats struct {
 	SceneNum int
 	StepNum  int
 }
+
+type AutoTestSceneCount struct {
+	Count int
+}
+
+type AutoTestSceneStepCount struct {
+	Count int
+}

modules/dop/component-protocol/components/test-dashboard/at_api_chart_group/at_api_avg_cost_chart/render.go

@@ -16,7 +16,6 @@ package at_api_avg_cost_chart
 
 import (
 	"context"
-	"sort"
 	"strconv"
 
 	"github.com/erda-project/erda-infra/base/servicehub"
@@ -47,17 +46,6 @@ func (f *Chart) Render(ctx context.Context, c *cptype.Component, scenario cptype
 	h := gshelper.NewGSHelper(gs)
 	atSvc := ctx.Value(types.AutoTestPlanService).(*autotestv2.Service)
 
-	sceneSteps, err := atSvc.ListAutoTestSceneSteps(h.GetGlobalAtSceneIDs())
-	if err != nil {
-		return err
-	}
-	sceneStepIDs := make([]uint64, 0, len(sceneSteps))
-	sceneStepMap := make(map[uint64]string, 0)
-	for _, v := range sceneSteps {
-		sceneStepMap[v.ID] = v.Name
-		sceneStepIDs = append(sceneStepIDs, v.ID)
-	}
-
 	timeFilter := h.GetAtSceneAndApiTimeFilter()
 	projectID, _ := strconv.ParseUint(cputil.GetInParamByKey(ctx, "projectId").(string), 10, 64)
 	costTimeAvg, err := atSvc.ExecHistoryApiAvgCostTime(apistructs.StatisticsExecHistoryRequest{
@@ -73,20 +61,17 @@ func (f *Chart) Render(ctx context.Context, c *cptype.Component, scenario cptype
 	if err != nil {
 		return err
 	}
-	sort.Slice(costTimeAvg, func(i, j int) bool {
-		return costTimeAvg[i].Avg > costTimeAvg[j].Avg
-	})
 
 	var (
 		values     []int64
 		categories []string
 	)
 	for _, v := range costTimeAvg {
-		if _, ok := sceneStepMap[v.StepID]; !ok {
-			continue
+		if v.Avg <= 0 {
+			v.Avg = 0
 		}
 		values = append(values, int64(v.Avg))
-		categories = append(categories, sceneStepMap[v.StepID])
+		categories = append(categories, v.Name)
 	}
 
 	c.Props = common.NewBarProps(values, categories, cputil.I18n(ctx, "api-avg-cost"), "{value}s")

modules/dop/component-protocol/components/test-dashboard/at_api_chart_group/at_api_rate_failed_chart/render.go

@@ -47,17 +47,6 @@ func (f *Chart) Render(ctx context.Context, c *cptype.Component, scenario cptype
 	h := gshelper.NewGSHelper(gs)
 	atSvc := ctx.Value(types.AutoTestPlanService).(*autotestv2.Service)
 
-	sceneSteps, err := atSvc.ListAutoTestSceneSteps(h.GetGlobalAtSceneIDs())
-	if err != nil {
-		return err
-	}
-	sceneStepIDs := make([]uint64, 0, len(sceneSteps))
-	sceneStepMap := make(map[uint64]string, 0)
-	for _, v := range sceneSteps {
-		sceneStepMap[v.ID] = v.Name
-		sceneStepIDs = append(sceneStepIDs, v.ID)
-	}
-
 	timeFilter := h.GetAtSceneAndApiTimeFilter()
 	projectID, _ := strconv.ParseUint(cputil.GetInParamByKey(ctx, "projectId").(string), 10, 64)
 	statusCounts, err := atSvc.ExecHistoryApiStatusCount(apistructs.StatisticsExecHistoryRequest{
@@ -89,12 +78,12 @@ func (f *Chart) Render(ctx context.Context, c *cptype.Component, scenario cptype
 		values     []int64
 		categories []string
 	)
-	for _, v := range statusCounts {
-		if _, ok := sceneStepMap[v.StepID]; !ok {
-			continue
+	for i, v := range statusCounts {
+		if i == 500 {
+			break
 		}
 		values = append(values, int64(v.FailRate))
-		categories = append(categories, sceneStepMap[v.StepID])
+		categories = append(categories, v.Name)
 	}
 
 	c.Props = common.NewBarProps(values, categories, cputil.I18n(ctx, "api-failed-rate"), "{value}%")

modules/dop/component-protocol/components/test-dashboard/common/gshelper/helper.go

@@ -214,22 +214,6 @@ func (h *GSHelper) GetGlobalAtStep() []apistructs.TestPlanV2Step {
 	return res
 }
 
-func (h *GSHelper) SetGlobalAtSceneIDs(ids []uint64) {
-	if h.gs == nil {
-		return
-	}
-	(*h.gs)["GlobalAtSceneIDs"] = ids
-}
-
-func (h *GSHelper) GetGlobalAtSceneIDs() []uint64 {
-	if h.gs == nil {
-		return nil
-	}
-	res := make([]uint64, 0)
-	_ = assign((*h.gs)["GlobalAtSceneIDs"], &res)
-	return res
-}
-
 type AtSceneAndApiTimeFilter struct {
 	TimeStart string `json:"timeStart"`
 	TimeEnd   string `json:"timeEnd"`

modules/dop/component-protocol/components/test-dashboard/filter/render.go

@@ -142,7 +142,7 @@ func (f *Filter) Render(ctx context.Context, c *cptype.Component, scenario cptyp
 	}())
 
 	//  set global auto test scene and step
-	if err = f.SetGlobalAtSceneAndStep(h); err != nil {
+	if err = f.SetGlobalAtStep(h); err != nil {
 		return err
 	}
 
@@ -158,30 +158,13 @@ func (f *Filter) Render(ctx context.Context, c *cptype.Component, scenario cptyp
 	return nil
 }
 
-func (f *Filter) SetGlobalAtSceneAndStep(h *gshelper.GSHelper) error {
+func (f *Filter) SetGlobalAtStep(h *gshelper.GSHelper) error {
 	steps, err := f.atTestPlan.ListStepByPlanID(h.GetGlobalAutoTestPlanIDs()...)
 	if err != nil {
 		return err
 	}
 
-	scenes, _, err := f.atTestPlan.ListSceneBySceneSetID(func() []uint64 {
-		setIDs := make([]uint64, 0, len(steps))
-		for _, v := range steps {
-			setIDs = append(setIDs, v.SceneSetID)
-		}
-		return setIDs
-	}()...)
-	if err != nil {
-		return err
-	}
-
-	sceneIDs := make([]uint64, 0, len(scenes))
-	for _, v := range scenes {
-		sceneIDs = append(sceneIDs, v.ID)
-	}
-
 	h.SetGlobalAtStep(steps)
-	h.SetGlobalAtSceneIDs(sceneIDs)
 	return nil
 }
 

modules/dop/component-protocol/components/test-dashboard/overview_group/blocks/at/at_num_api/render.go

@@ -43,35 +43,20 @@ func (t *Text) Render(ctx context.Context, c *cptype.Component, scenario cptype.
 	h := gshelper.NewGSHelper(gs)
 	atSvc := ctx.Value(types.AutoTestPlanService).(*autotestv2.Service)
 
-	scenes, _, err := atSvc.ListSceneBySceneSetID(func() []uint64 {
-		setIDs := make([]uint64, 0, len(h.GetBlockAtStep()))
-		for _, v := range h.GetBlockAtStep() {
-			setIDs = append(setIDs, v.SceneSetID)
+	stepCount, err := atSvc.CountAutoTestSceneSteps(func() []uint64 {
+		selectPlans := h.GetAtBlockFilterTestPlanList()
+		selectPlanIDs := make([]uint64, 0, len(selectPlans))
+		for _, v := range selectPlans {
+			selectPlanIDs = append(selectPlanIDs, v.ID)
 		}
-		return setIDs
-	}()...)
-	if err != nil {
-		return err
-	}
-	sceneSteps, err := atSvc.ListAutoTestSceneSteps(func() []uint64 {
-		sceneIDs := make([]uint64, 0, len(scenes))
-		for _, v := range scenes {
-			sceneIDs = append(sceneIDs, v.ID)
-		}
-		return sceneIDs
+		return selectPlanIDs
 	}())
 	if err != nil {
 		return err
 	}
 	tv := pkg.TextValue{
 		Value: strutil.String(func() int {
-			var count int
-			for _, v := range sceneSteps {
-				if v.Type.IsEffectiveStepType() && v.Name != "" {
-					count++
-				}
-			}
-			return count
+			return stepCount.Count
 		}()),
 		Kind: cputil.I18n(ctx, "auto-test-api-num"),
 	}

modules/dop/component-protocol/components/test-dashboard/overview_group/blocks/at/at_num_scene/render.go

@@ -43,20 +43,21 @@ func (t *Text) Render(ctx context.Context, c *cptype.Component, scenario cptype.
 	h := gshelper.NewGSHelper(gs)
 	atSvc := ctx.Value(types.AutoTestPlanService).(*autotestv2.Service)
 
-	_, total, err := atSvc.ListSceneBySceneSetID(func() []uint64 {
-		setIDs := make([]uint64, 0, len(h.GetBlockAtStep()))
-		for _, v := range h.GetBlockAtStep() {
-			setIDs = append(setIDs, v.SceneSetID)
+	sceneCount, err := atSvc.CountSceneByPlanIDs(func() []uint64 {
+		selectPlans := h.GetAtBlockFilterTestPlanList()
+		selectPlanIDs := make([]uint64, 0, len(selectPlans))
+		for _, v := range selectPlans {
+			selectPlanIDs = append(selectPlanIDs, v.ID)
 		}
-		return setIDs
+		return selectPlanIDs
 	}()...)
 	if err != nil {
 		return err
 	}
 
 	tv := pkg.TextValue{
 		Value: strutil.String(func() int {
-			return int(total)
+			return sceneCount.Count
 		}()),
 		Kind: cputil.I18n(ctx, "auto-test-scene-num"),
 	}

modules/dop/component-protocol/components/test-dashboard/scene_chart_group/scene_avg_cost_chart/render.go

@@ -16,7 +16,6 @@ package scene_avg_cost_chart
 
 import (
 	"context"
-	"sort"
 	"strconv"
 	"time"
 
@@ -48,25 +47,7 @@ func (f *Chart) Render(ctx context.Context, c *cptype.Component, scenario cptype
 	h := gshelper.NewGSHelper(gs)
 	atSvc := ctx.Value(types.AutoTestPlanService).(*autotestv2.Service)
 
-	scenes, _, err := atSvc.ListSceneBySceneSetID(func() []uint64 {
-		setIDs := make([]uint64, 0, len(h.GetGlobalAtStep()))
-		for _, v := range h.GetGlobalAtStep() {
-			setIDs = append(setIDs, v.SceneSetID)
-		}
-		return setIDs
-	}()...)
-	if err != nil {
-		return err
-	}
-	sceneIDs := make([]uint64, 0, len(scenes))
-	sceneMap := make(map[uint64]string, 0)
-	for _, v := range scenes {
-		sceneMap[v.ID] = v.Name
-		sceneIDs = append(sceneIDs, v.ID)
-	}
-
 	timeFilter := h.GetAtSceneAndApiTimeFilter()
-
 	projectID, _ := strconv.ParseUint(cputil.GetInParamByKey(ctx, "projectId").(string), 10, 64)
 	costTimeAvg, err := atSvc.ExecHistorySceneAvgCostTime(apistructs.StatisticsExecHistoryRequest{
 		TimeStart:    timeFilter.TimeStart,
@@ -81,20 +62,17 @@ func (f *Chart) Render(ctx context.Context, c *cptype.Component, scenario cptype
 	if err != nil {
 		return err
 	}
-	sort.Slice(costTimeAvg, func(i, j int) bool {
-		return costTimeAvg[i].Avg > costTimeAvg[j].Avg
-	})
 
 	var (
 		values     []int64
 		categories []string
 	)
 	for _, v := range costTimeAvg {
-		if _, ok := sceneMap[v.SceneID]; !ok {
-			continue
+		if v.Avg <= 0 {
+			v.Avg = 0
 		}
 		values = append(values, int64(v.Avg))
-		categories = append(categories, sceneMap[v.SceneID])
+		categories = append(categories, v.Name)
 	}
 
 	c.Props = common.NewBarProps(values, categories, cputil.I18n(ctx, "scene-avg-cost"), "{value}s")

modules/dop/component-protocol/components/test-dashboard/scene_chart_group/scene_rate_failed_chart/render.go

@@ -47,23 +47,6 @@ func (f *Chart) Render(ctx context.Context, c *cptype.Component, scenario cptype
 	h := gshelper.NewGSHelper(gs)
 	atSvc := ctx.Value(types.AutoTestPlanService).(*autotestv2.Service)
 
-	scenes, _, err := atSvc.ListSceneBySceneSetID(func() []uint64 {
-		setIDs := make([]uint64, 0, len(h.GetGlobalAtStep()))
-		for _, v := range h.GetGlobalAtStep() {
-			setIDs = append(setIDs, v.SceneSetID)
-		}
-		return setIDs
-	}()...)
-	if err != nil {
-		return err
-	}
-	sceneIDs := make([]uint64, 0, len(scenes))
-	sceneMap := make(map[uint64]string, 0)
-	for _, v := range scenes {
-		sceneMap[v.ID] = v.Name
-		sceneIDs = append(sceneIDs, v.ID)
-	}
-
 	timeFilter := h.GetAtSceneAndApiTimeFilter()
 	projectID, _ := strconv.ParseUint(cputil.GetInParamByKey(ctx, "projectId").(string), 10, 64)
 	statusCounts, err := atSvc.ExecHistorySceneStatusCount(apistructs.StatisticsExecHistoryRequest{
@@ -95,12 +78,12 @@ func (f *Chart) Render(ctx context.Context, c *cptype.Component, scenario cptype
 		values     []int64
 		categories []string
 	)
-	for _, v := range statusCounts {
-		if _, ok := sceneMap[v.SceneID]; !ok {
-			continue
+	for i, v := range statusCounts {
+		if i == 500 {
+			break
 		}
 		values = append(values, int64(v.FailRate))
-		categories = append(categories, sceneMap[v.SceneID])
+		categories = append(categories, v.Name)
 	}
 
 	c.Props = common.NewBarProps(values, categories, cputil.I18n(ctx, "scene-failed-rate"), "{value}%")