api:add api for history_hot_region info;cluster add storage for hot region;config add config for hot-region storage

cmd/pd-server/docs/docs.go

@@ -0,0 +1,74 @@
+// GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
+// This file was generated by swaggo/swag
+
+package docs
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+
+	"github.com/alecthomas/template"
+	"github.com/swaggo/swag"
+)
+
+var doc = `{
+    "schemes": {{ marshal .Schemes }},
+    "swagger": "2.0",
+    "info": {
+        "description": "{{.Description}}",
+        "title": "{{.Title}}",
+        "contact": {},
+        "version": "{{.Version}}"
+    },
+    "host": "{{.Host}}",
+    "basePath": "{{.BasePath}}",
+    "paths": {}
+}`
+
+type swaggerInfo struct {
+	Version     string
+	Host        string
+	BasePath    string
+	Schemes     []string
+	Title       string
+	Description string
+}
+
+// SwaggerInfo holds exported Swagger Info so clients can modify it
+var SwaggerInfo = swaggerInfo{
+	Version:     "",
+	Host:        "",
+	BasePath:    "",
+	Schemes:     []string{},
+	Title:       "",
+	Description: "",
+}
+
+type s struct{}
+
+func (s *s) ReadDoc() string {
+	sInfo := SwaggerInfo
+	sInfo.Description = strings.Replace(sInfo.Description, "\n", "\\n", -1)
+
+	t, err := template.New("swagger_info").Funcs(template.FuncMap{
+		"marshal": func(v interface{}) string {
+			a, _ := json.Marshal(v)
+			return string(a)
+		},
+	}).Parse(doc)
+	if err != nil {
+		return doc
+	}
+
+	var tpl bytes.Buffer
+	if err := t.Execute(&tpl, sInfo); err != nil {
+		return doc
+	}
+
+	return tpl.String()
+}
+
+func init() {
+	swag.Register(swag.Name, &s{})
+}

cmd/pd-server/docs/swagger.json

@@ -0,0 +1,7 @@
+{
+    "swagger": "2.0",
+    "info": {
+        "contact": {}
+    },
+    "paths": {}
+}

cmd/pd-server/docs/swagger.yaml

@@ -0,0 +1,4 @@
+info:
+  contact: {}
+paths: {}
+swagger: "2.0"

server/api/hot_status_test.go

@@ -197,6 +197,5 @@ func writeToDB(kv *kv.LeveldbKV, hotRegions []*core.HistoryHotRegion) error {
 		}
 		batch.Put([]byte(key), value)
 	}
-	kv.Write(batch, nil)
-	return nil
+	return kv.Write(batch, nil)
 }

server/api/router.go

@@ -153,6 +153,7 @@ func createRouter(prefix string, svr *server.Server) *mux.Router {
 	hotStatusHandler := newHotStatusHandler(handler, rd)
 	apiRouter.HandleFunc("/hotspot/regions/write", hotStatusHandler.GetHotWriteRegions).Methods("GET")
 	apiRouter.HandleFunc("/hotspot/regions/read", hotStatusHandler.GetHotReadRegions).Methods("GET")
+	apiRouter.HandleFunc("/hotspot/regions/history", hotStatusHandler.GetHistoryHotRegions).Methods("GET")
 	apiRouter.HandleFunc("/hotspot/stores", hotStatusHandler.GetHotStores).Methods("GET")
 	apiRouter.HandleFunc("/hotspot/regions/history", hotStatusHandler.GetHistoryHotRegions).Methods("GET")
 

server/core/hot_region_storage.go

@@ -100,10 +100,30 @@ const (
 	defaultDeleteTime = 4
 )
 
+// HotRegionType stands for hot type.
+type HotRegionType uint32
+
+// Flags for flow.
+const (
+	WriteType HotRegionType = iota
+	ReadType
+)
+
 // HotRegionTypes stands for hot type.
 var HotRegionTypes = []string{
-	"read",
-	"write",
+	WriteType.String(),
+	ReadType.String(),
+}
+
+// String return HotRegionType in string format.
+func (h HotRegionType) String() string {
+	switch h {
+	case WriteType:
+		return "write"
+	case ReadType:
+		return "read"
+	}
+	return "unimplemented"
 }
 
 // NewHotRegionsStorage create storage to store hot regions info.
@@ -223,14 +243,14 @@ func (h *HotRegionStorage) pullHotRegionInfo() error {
 	if err != nil {
 		return err
 	}
-	if err := h.packHistoryHotRegions(historyHotReadRegions, HotRegionTypes[0]); err != nil {
+	if err := h.packHistoryHotRegions(historyHotReadRegions, ReadType.String()); err != nil {
 		return err
 	}
 	historyHotWriteRegions, err := h.hotRegionStorageHandler.PackHistoryHotWriteRegions()
 	if err != nil {
 		return err
 	}
-	err = h.packHistoryHotRegions(historyHotWriteRegions, HotRegionTypes[1])
+	err = h.packHistoryHotRegions(historyHotWriteRegions, WriteType.String())
 	return err
 }
 
@@ -248,7 +268,7 @@ func (h *HotRegionStorage) packHistoryHotRegions(historyHotRegions []HistoryHotR
 		}
 		historyHotRegions[i].StartKey = region.StartKey
 		historyHotRegions[i].EndKey = region.EndKey
-		key := HotRegionStorePath(HotRegionTypes[0], historyHotRegions[i].UpdateTime, historyHotRegions[i].RegionID)
+		key := HotRegionStorePath(hotRegionType, historyHotRegions[i].UpdateTime, historyHotRegions[i].RegionID)
 		h.batchHotInfo[key] = &historyHotRegions[i]
 	}
 	return nil

server/core/hot_region_storage_test.go

@@ -102,25 +102,33 @@ func (t *testHotRegionStorage) TestHotRegionWrite(c *C) {
 			UpdateTime:    now.UnixNano() / int64(time.Millisecond),
 			RegionID:      1,
 			StoreID:       1,
-			HotRegionType: HotRegionTypes[0],
+			HotRegionType: ReadType.String(),
 		},
 		{
 			UpdateTime:    now.Add(10*time.Second).UnixNano() / int64(time.Millisecond),
 			RegionID:      2,
 			StoreID:       1,
-			HotRegionType: HotRegionTypes[0],
+			HotRegionType: ReadType.String(),
 		},
 		{
 			UpdateTime:    now.Add(20*time.Second).UnixNano() / int64(time.Millisecond),
 			RegionID:      3,
 			StoreID:       1,
-			HotRegionType: HotRegionTypes[0],
+			HotRegionType: ReadType.String(),
 		},
 	}
 	packHotRegionInfo.historyHotReads = hotRegionStorages
+	packHotRegionInfo.historyHotWrites = []HistoryHotRegion{
+		{
+			UpdateTime:    now.Add(30*time.Second).UnixNano() / int64(time.Millisecond),
+			RegionID:      4,
+			StoreID:       1,
+			HotRegionType: WriteType.String(),
+		},
+	}
 	store.pullHotRegionInfo()
 	store.flush()
-	iter := store.NewIterator(HotRegionTypes,
+	iter := store.NewIterator([]string{ReadType.String()},
 		now.UnixNano()/int64(time.Millisecond),
 		now.Add(40*time.Second).UnixNano()/int64(time.Millisecond))
 	index := 0
@@ -143,19 +151,19 @@ func (t *testHotRegionStorage) TestHotRegionDelete(c *C) {
 			UpdateTime:    deleteDate.UnixNano() / int64(time.Millisecond),
 			RegionID:      1,
 			StoreID:       1,
-			HotRegionType: HotRegionTypes[0],
+			HotRegionType: ReadType.String(),
 		},
 		{
 			UpdateTime:    deleteDate.Add(10*time.Second).UnixNano() / int64(time.Millisecond),
 			RegionID:      2,
 			StoreID:       1,
-			HotRegionType: HotRegionTypes[0],
+			HotRegionType: ReadType.String(),
 		},
 		{
 			UpdateTime:    time.Now().UnixNano() / int64(time.Millisecond),
 			RegionID:      3,
 			StoreID:       1,
-			HotRegionType: HotRegionTypes[0],
+			HotRegionType: ReadType.String(),
 		},
 	}
 	packHotRegionInfo.historyHotReads = hotRegionStorages

server/handler.go

@@ -912,54 +912,51 @@ func (h *Handler) IsLeader() bool {
 
 // PackHistoryHotReadRegions get read hot region info in HistoryHotRegion form.
 func (h *Handler) PackHistoryHotReadRegions() (historyHotRegions []core.HistoryHotRegion, err error) {
-	hotReadLeaderRegions := h.GetHotReadRegions().AsLeader
-	historyLeaderHotRegions, err := h.packHotRegions(hotReadLeaderRegions, true, core.HotRegionTypes[0])
-	if err != nil {
+	hotReadRegions := h.GetHotReadRegions()
+	if hotReadRegions == nil {
 		return
 	}
-	hotReadPeerRegions := h.GetHotReadRegions().AsPeer
-	historyPeerHotRegions, err := h.packHotRegions(hotReadPeerRegions, false, core.HotRegionTypes[0])
+	hotReadPeerRegions := hotReadRegions.AsPeer
+	historyPeerHotRegions, err := h.packHotRegions(hotReadPeerRegions, core.ReadType.String())
 	if err != nil {
 		return
 	}
-	historyHotRegions = append(historyHotRegions, historyLeaderHotRegions...)
 	historyHotRegions = append(historyHotRegions, historyPeerHotRegions...)
 	return
 }
 
 // PackHistoryHotWriteRegions get write hot region info in HistoryHotRegion from
 func (h *Handler) PackHistoryHotWriteRegions() (historyHotRegions []core.HistoryHotRegion, err error) {
-	hotWriteLeaderRegions := h.GetHotWriteRegions().AsLeader
-	historyLeaderHotRegions, err := h.packHotRegions(hotWriteLeaderRegions, true, core.HotRegionTypes[1])
-	if err != nil {
+	hotWriteRegions := h.GetHotWriteRegions()
+	if hotWriteRegions == nil {
 		return
 	}
-	hotWritePeerRegions := h.GetHotWriteRegions().AsPeer
-	historyPeerHotRegions, err := h.packHotRegions(hotWritePeerRegions, false, core.HotRegionTypes[1])
+	hotWritePeerRegions := hotWriteRegions.AsPeer
+	historyPeerHotRegions, err := h.packHotRegions(hotWritePeerRegions, core.WriteType.String())
 	if err != nil {
 		return
 	}
-	historyHotRegions = append(historyHotRegions, historyLeaderHotRegions...)
 	historyHotRegions = append(historyHotRegions, historyPeerHotRegions...)
 	return
 }
 
-func (h *Handler) packHotRegions(hotPeersStat statistics.StoreHotPeersStat, isLeader bool, hotRegionType string) (historyHotRegions []core.HistoryHotRegion, err error) {
+func (h *Handler) packHotRegions(hotPeersStat statistics.StoreHotPeersStat, hotRegionType string) (historyHotRegions []core.HistoryHotRegion, err error) {
 	c, err := h.GetRaftCluster()
 	if err != nil {
 		return nil, err
 	}
 	for _, hotPeersStat := range hotPeersStat {
 		stats := hotPeersStat.Stats
 		for _, hotPeerStat := range stats {
-			region := c.GetRegion(hotPeerStat.RegionID).GetMeta()
-			region, err := encryption.EncryptRegion(region, h.s.encryptionKeyManager)
+			region := c.GetRegion(hotPeerStat.RegionID)
+			meta := region.GetMeta()
+			meta, err := encryption.EncryptRegion(meta, h.s.encryptionKeyManager)
 			if err != nil {
 				return nil, err
 			}
 			var peerID uint64
 			var isLearner bool
-			for _, peer := range region.Peers {
+			for _, peer := range meta.Peers {
 				if peer.StoreId == hotPeerStat.StoreID {
 					peerID = peer.Id
 					isLearner = peer.Role == metapb.PeerRole_Learner
@@ -971,15 +968,15 @@ func (h *Handler) packHotRegions(hotPeersStat statistics.StoreHotPeersStat, isLe
 				RegionID:       hotPeerStat.RegionID,
 				StoreID:        hotPeerStat.StoreID,
 				PeerID:         peerID,
-				IsLeader:       isLeader,
+				IsLeader:       meta.Id == region.GetLeader().Id,
 				IsLearner:      isLearner,
 				HotDegree:      int64(hotPeerStat.HotDegree),
 				FlowBytes:      hotPeerStat.ByteRate,
 				KeyRate:        hotPeerStat.KeyRate,
 				QueryRate:      hotPeerStat.QueryRate,
-				StartKey:       region.StartKey,
-				EndKey:         region.EndKey,
-				EncryptionMeta: region.EncryptionMeta,
+				StartKey:       meta.StartKey,
+				EndKey:         meta.EndKey,
+				EncryptionMeta: meta.EncryptionMeta,
 				HotRegionType:  hotRegionType,
 			}
 			historyHotRegions = append(historyHotRegions, stat)