refactor: Switch to use ziputil, netutil, reflectutil and fileswap (p…

…ingcap#1067)

.golangci.yml

@@ -2,6 +2,7 @@ run:
   skip-dirs:
     - swaggerspec
     - pkg/uiserver
+    - ui
   timeout: 2m
 
 issues:

pkg/apiserver/clusterinfo/hostinfo/cluster_config.go

@@ -7,7 +7,7 @@ import (
 
 	"gorm.io/gorm"
 
-	"github.com/pingcap/tidb-dashboard/pkg/utils/host"
+	"github.com/pingcap/tidb-dashboard/util/netutil"
 )
 
 type clusterConfigModel struct {
@@ -29,7 +29,7 @@ func FillInstances(db *gorm.DB, m InfoMap) error {
 	}
 
 	for _, row := range rows {
-		hostname, _, err := host.ParseHostAndPortFromAddress(row.Instance)
+		hostname, _, err := netutil.ParseHostAndPortFromAddress(row.Instance)
 		if err != nil {
 			continue
 		}

pkg/apiserver/clusterinfo/hostinfo/cluster_hardware.go

@@ -9,7 +9,7 @@ import (
 
 	"gorm.io/gorm"
 
-	"github.com/pingcap/tidb-dashboard/pkg/utils/host"
+	"github.com/pingcap/tidb-dashboard/util/netutil"
 )
 
 // Used to deserialize from JSON_VALUE.
@@ -46,7 +46,7 @@ func FillFromClusterHardwareTable(db *gorm.DB, m InfoMap) error {
 	tiFlashDisks := make([]clusterTableModel, 0)
 
 	for _, row := range rows {
-		hostname, _, err := host.ParseHostAndPortFromAddress(row.Instance)
+		hostname, _, err := netutil.ParseHostAndPortFromAddress(row.Instance)
 		if err != nil {
 			continue
 		}
@@ -126,7 +126,7 @@ func FillFromClusterHardwareTable(db *gorm.DB, m InfoMap) error {
 	}
 	// Back fill TiFlash instances
 	for instance, de := range tiFlashDiskInfo {
-		hostname, _, err := host.ParseHostAndPortFromAddress(instance)
+		hostname, _, err := netutil.ParseHostAndPortFromAddress(instance)
 		if err != nil {
 			panic(err)
 		}

pkg/apiserver/clusterinfo/hostinfo/cluster_load.go

@@ -8,7 +8,7 @@ import (
 
 	"gorm.io/gorm"
 
-	"github.com/pingcap/tidb-dashboard/pkg/utils/host"
+	"github.com/pingcap/tidb-dashboard/util/netutil"
 )
 
 // Used to deserialize from JSON_VALUE.
@@ -40,7 +40,7 @@ func FillFromClusterLoadTable(db *gorm.DB, m InfoMap) error {
 	}
 
 	for _, row := range rows {
-		hostname, _, err := host.ParseHostAndPortFromAddress(row.Instance)
+		hostname, _, err := netutil.ParseHostAndPortFromAddress(row.Instance)
 		if err != nil {
 			continue
 		}

pkg/apiserver/debugapi/service.go

@@ -14,12 +14,8 @@ import (
 
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/debugapi/endpoint"
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/user"
-	"github.com/pingcap/tidb-dashboard/pkg/apiserver/utils"
 	"github.com/pingcap/tidb-dashboard/util/rest"
-)
-
-const (
-	tokenIssuer = "debugAPI"
+	"github.com/pingcap/tidb-dashboard/util/rest/fileswap"
 )
 
 func registerRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
@@ -34,11 +30,13 @@ func registerRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
 
 type Service struct {
 	Client *endpoint.Client
+	fSwap  *fileswap.Handler
 }
 
 func newService(hp httpClientParam) *Service {
 	return &Service{
 		Client: endpoint.NewClient(newHTTPClient(hp), endpointDefs),
+		fSwap:  fileswap.New(),
 	}
 }
 
@@ -82,27 +80,31 @@ func (s *Service) RequestEndpoint(c *gin.Context) {
 	}
 	defer res.Response.Body.Close() //nolint:errcheck
 
-	ext := getExtFromContentTypeHeader(res.Header.Get("Content-Type"))
-	fileName := fmt.Sprintf("%s_%d%s", req.EndpointID, time.Now().Unix(), ext)
-
-	writer, token, err := utils.FSPersist(utils.FSPersistConfig{
-		TokenIssuer:      tokenIssuer,
-		TokenExpire:      time.Minute * 5, // Note: the expire time should include request time.
-		TempFilePattern:  "debug_api",
-		DownloadFileName: fileName,
-	})
+	writer, err := s.fSwap.NewFileWriter("debug_api")
 	if err != nil {
 		_ = c.Error(err)
 		return
 	}
-	defer writer.Close() //nolint:errcheck
+	defer func() {
+		_ = writer.Close()
+	}()
+
 	_, err = io.Copy(writer, res.Response.Body)
 	if err != nil {
 		_ = c.Error(err)
 		return
 	}
 
-	c.String(http.StatusOK, token)
+	ext := getExtFromContentTypeHeader(res.Header.Get("Content-Type"))
+	fileName := fmt.Sprintf("%s_%d%s", req.EndpointID, time.Now().Unix(), ext)
+	downloadToken, err := writer.GetDownloadToken(fileName, time.Minute*5)
+	if err != nil {
+		// This shall never happen
+		_ = c.Error(err)
+		return
+	}
+
+	c.String(http.StatusOK, downloadToken)
 }
 
 // @Summary Download a finished request result
@@ -112,8 +114,7 @@ func (s *Service) RequestEndpoint(c *gin.Context) {
 // @Failure 500 {object} rest.ErrorResponse
 // @Router /debug_api/download [get]
 func (s *Service) Download(c *gin.Context) {
-	token := c.Query("token")
-	utils.FSServe(c, token, tokenIssuer)
+	s.fSwap.HandleDownloadRequest(c)
 }
 
 // @Summary Get all endpoints

pkg/apiserver/logsearch/pack.go

@@ -9,8 +9,8 @@ import (
 	"github.com/pingcap/log"
 	"go.uber.org/zap"
 
-	"github.com/pingcap/tidb-dashboard/pkg/apiserver/utils"
 	"github.com/pingcap/tidb-dashboard/util/rest"
+	"github.com/pingcap/tidb-dashboard/util/ziputil"
 )
 
 func serveTaskForDownload(task *TaskModel, c *gin.Context) {
@@ -41,7 +41,7 @@ func serveMultipleTaskForDownload(tasks []*TaskModel, c *gin.Context) {
 
 	c.Writer.Header().Set("Content-type", "application/octet-stream")
 	c.Writer.Header().Set("Content-Disposition", "attachment; filename=\"logs.zip\"")
-	err := utils.StreamZipPack(c.Writer, filePaths, false)
+	err := ziputil.WriteZipFromFiles(c.Writer, filePaths, false)
 	if err != nil {
 		log.Error("Stream zip pack failed", zap.Error(err))
 	}

pkg/apiserver/profiling/router.go

@@ -17,6 +17,7 @@ import (
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/utils"
 	"github.com/pingcap/tidb-dashboard/pkg/config"
 	"github.com/pingcap/tidb-dashboard/util/rest"
+	"github.com/pingcap/tidb-dashboard/util/ziputil"
 )
 
 // Register register the handlers to the service.
@@ -223,7 +224,7 @@ func (s *Service) downloadGroup(c *gin.Context) {
 	fileName := fmt.Sprintf("profiling_pack_%d.zip", taskGroupID)
 	c.Writer.Header().Set("Content-type", "application/octet-stream")
 	c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName))
-	err = utils.StreamZipPack(c.Writer, filePathes, true)
+	err = ziputil.WriteZipFromFiles(c.Writer, filePathes, true)
 	if err != nil {
 		log.Error("Stream zip pack failed", zap.Error(err))
 	}
@@ -262,7 +263,7 @@ func (s *Service) downloadSingle(c *gin.Context) {
 	fileName := fmt.Sprintf("profiling_%d.zip", taskID)
 	c.Writer.Header().Set("Content-type", "application/octet-stream")
 	c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName))
-	err = utils.StreamZipPack(c.Writer, []string{task.FilePath}, true)
+	err = ziputil.WriteZipFromFiles(c.Writer, []string{task.FilePath}, true)
 	if err != nil {
 		log.Error("Stream zip pack failed", zap.Error(err))
 	}

pkg/apiserver/slowquery/model.go

@@ -6,6 +6,7 @@ import (
 	"github.com/thoas/go-funk"
 
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/utils"
+	"github.com/pingcap/tidb-dashboard/util/reflectutil"
 )
 
 type Model struct {
@@ -96,7 +97,7 @@ type Field struct {
 }
 
 func getFieldsAndTags() (slowQueryFields []Field) {
-	fields := utils.GetFieldsAndTags(Model{}, []string{"gorm", "proj", "json"})
+	fields := reflectutil.GetFieldsAndTags(Model{}, []string{"gorm", "proj", "json"})
 
 	for _, f := range fields {
 		sqf := Field{

pkg/apiserver/statement/models.go

@@ -10,6 +10,7 @@ import (
 	"gorm.io/gorm/schema"
 
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/utils"
+	"github.com/pingcap/tidb-dashboard/util/reflectutil"
 )
 
 // TimeRange represents a range of time.
@@ -135,7 +136,7 @@ type Field struct {
 var gormDefaultNamingStrategy = schema.NamingStrategy{}
 
 func getFieldsAndTags() (stmtFields []Field) {
-	fields := utils.GetFieldsAndTags(Model{}, []string{"related", "agg", "json"})
+	fields := reflectutil.GetFieldsAndTags(Model{}, []string{"related", "agg", "json"})
 
 	for _, f := range fields {
 		sf := Field{