api: fix scan regions (tikv#1648)

* fix scan regions bug: [s,d), s < key < d not found

* add one more test

* cleanup go.mod go.sum

* change semantic of ScanRange* from 'contain' to 'intersect'

server/api/region_test.go

@@ -269,20 +269,40 @@ func (s *testGetRegionSuite) TestRegionKey(c *C) {
 func (s *testGetRegionSuite) TestScanRegionByKey(c *C) {
 	r1 := newTestRegionInfo(2, 1, []byte("a"), []byte("b"))
 	r2 := newTestRegionInfo(3, 1, []byte("b"), []byte("c"))
-	r3 := newTestRegionInfo(4, 2, []byte("c"), []byte("d"))
+	r3 := newTestRegionInfo(4, 2, []byte("c"), []byte("e"))
+	r4 := newTestRegionInfo(5, 2, []byte("x"), []byte("z"))
 	r := newTestRegionInfo(99, 1, []byte{0xFF, 0xFF, 0xAA}, []byte{0xFF, 0xFF, 0xCC}, core.SetWrittenBytes(500), core.SetReadBytes(800), core.SetRegionConfVer(3), core.SetRegionVersion(2))
 	mustRegionHeartbeat(c, s.svr, r1)
 	mustRegionHeartbeat(c, s.svr, r2)
 	mustRegionHeartbeat(c, s.svr, r3)
+	mustRegionHeartbeat(c, s.svr, r4)
 	mustRegionHeartbeat(c, s.svr, r)
 
 	url := fmt.Sprintf("%s/regions/key?key=%s", s.urlPrefix, "b")
-	regionIds := []uint64{3, 4, 99}
+	regionIds := []uint64{3, 4, 5, 99}
 	regions := &RegionsInfo{}
 	err := readJSONWithURL(url, regions)
 	c.Assert(err, IsNil)
 	c.Assert(len(regionIds), Equals, regions.Count)
 	for i, v := range regionIds {
 		c.Assert(v, Equals, regions.Regions[i].ID)
 	}
+	url = fmt.Sprintf("%s/regions/key?key=%s", s.urlPrefix, "d")
+	regionIds = []uint64{4, 5, 99}
+	regions = &RegionsInfo{}
+	err = readJSONWithURL(url, regions)
+	c.Assert(err, IsNil)
+	c.Assert(len(regionIds), Equals, regions.Count)
+	for i, v := range regionIds {
+		c.Assert(v, Equals, regions.Regions[i].ID)
+	}
+	url = fmt.Sprintf("%s/regions/key?key=%s", s.urlPrefix, "g")
+	regionIds = []uint64{5, 99}
+	regions = &RegionsInfo{}
+	err = readJSONWithURL(url, regions)
+	c.Assert(err, IsNil)
+	c.Assert(len(regionIds), Equals, regions.Count)
+	for i, v := range regionIds {
+		c.Assert(v, Equals, regions.Regions[i].ID)
+	}
 }

server/api/stats_test.go

@@ -147,6 +147,14 @@ func (s *testStatsSuite) TestRegionStats(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(stats, DeepEquals, statsAll)
 
+	args := fmt.Sprintf("?start_key=%s&end_key=%s", url.QueryEscape("\x01\x02"), url.QueryEscape("xyz\x00\x00"))
+	res, err = http.Get(statsURL + args)
+	c.Assert(err, IsNil)
+	stats = &statistics.RegionStats{}
+	err = apiutil.ReadJSON(res.Body, stats)
+	c.Assert(err, IsNil)
+	c.Assert(stats, DeepEquals, statsAll)
+
 	stats23 := &statistics.RegionStats{
 		Count:            2,
 		EmptyCount:       1,
@@ -159,7 +167,8 @@ func (s *testStatsSuite) TestRegionStats(c *C) {
 		StorePeerSize:    map[uint64]int64{1: 201, 4: 200, 5: 201},
 		StorePeerKeys:    map[uint64]int64{1: 151, 4: 150, 5: 151},
 	}
-	args := fmt.Sprintf("?start_key=%s&end_key=%s", url.QueryEscape("\x01\x02"), url.QueryEscape("xyz\x00\x00"))
+
+	args = fmt.Sprintf("?start_key=%s&end_key=%s", url.QueryEscape("a"), url.QueryEscape("x"))
 	res, err = http.Get(statsURL + args)
 	c.Assert(err, IsNil)
 	stats = &statistics.RegionStats{}

server/core/region.go

@@ -693,7 +693,8 @@ func (r *RegionsInfo) GetFollower(storeID uint64, regionID uint64) *RegionInfo {
 	return r.followers[storeID].Get(regionID)
 }
 
-// ScanRange scans region with start key, until number greater than limit.
+// ScanRange scans from the first region containing or behind start key,
+// until number greater than limit.
 func (r *RegionsInfo) ScanRange(startKey []byte, limit int) []*RegionInfo {
 	res := make([]*RegionInfo, 0, limit)
 	r.tree.scanRange(startKey, func(metaRegion *metapb.Region) bool {
@@ -703,11 +704,11 @@ func (r *RegionsInfo) ScanRange(startKey []byte, limit int) []*RegionInfo {
 	return res
 }
 
-// ScanRangeWithEndKey scans region with start key and end key.
+// ScanRangeWithEndKey scans regions intersecting [start key, end key).
 func (r *RegionsInfo) ScanRangeWithEndKey(startKey, endKey []byte) []*RegionInfo {
 	var regions []*RegionInfo
 	r.tree.scanRange(startKey, func(meta *metapb.Region) bool {
-		if len(endKey) > 0 && (len(meta.EndKey) == 0 || bytes.Compare(meta.EndKey, endKey) >= 0) {
+		if len(endKey) > 0 && bytes.Compare(meta.StartKey, endKey) >= 0 {
 			return false
 		}
 		if region := r.GetRegion(meta.GetId()); region != nil {
@@ -718,7 +719,8 @@ func (r *RegionsInfo) ScanRangeWithEndKey(startKey, endKey []byte) []*RegionInfo
 	return regions
 }
 
-// ScanRangeWithIterator scans region with start key, until iterator returns false.
+// ScanRangeWithIterator scans from the first region containing or behind start key,
+// until iterator returns false.
 func (r *RegionsInfo) ScanRangeWithIterator(startKey []byte, iterator func(metaRegion *metapb.Region) bool) {
 	r.tree.scanRange(startKey, iterator)
 }

server/core/region_tree.go

@@ -159,8 +159,15 @@ func (t *regionTree) find(region *metapb.Region) *regionItem {
 	return result
 }
 
+// scanRage scans from the first region containing or behind the start key
+// until f return false
 func (t *regionTree) scanRange(startKey []byte, f func(*metapb.Region) bool) {
-	startItem := &regionItem{region: &metapb.Region{StartKey: startKey}}
+	region := &metapb.Region{StartKey: startKey}
+	// find if there is a region with key range [s, d), s < startKey < d
+	startItem := t.find(region)
+	if startItem == nil {
+		startItem = &regionItem{region: &metapb.Region{StartKey: startKey}}
+	}
 	t.tree.AscendGreaterOrEqual(startItem, func(item btree.Item) bool {
 		return f(item.(*regionItem).region)
 	})

server/statistics/region.go

@@ -113,7 +113,7 @@ func (s *RegionStats) Observe(r *core.RegionInfo) {
 	}
 }
 
-// GetRegionStats scans regions that inside range [startKey, endKey) and sums up
+// GetRegionStats scans regions that intersect range [startKey, endKey) and sums up
 // their statistics.
 func GetRegionStats(r *core.RegionsInfo, startKey, endKey []byte) *RegionStats {
 	stats := newRegionStats()