Skip to content

Commit

Permalink
done
Browse files Browse the repository at this point in the history
  • Loading branch information
@wjhuang2016
wjhuang2016 committed Dec 16, 2021
1 parent 5d62c2a commit 2fd3b08
Show file tree
Hide file tree
Showing 22 changed files with 201 additions and 100 deletions.
9 changes: 6 additions & 3 deletions distsql/request_builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/codec"
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/ranger"
"github.com/pingcap/tipb/go-tipb"
Expand Down Expand Up @@ -382,7 +383,7 @@ func tablesRangesToKVRanges(tids []int64, ranges []*ranger.Range, fb *statistics
// since we need to guarantee each range falls inside the exactly one bucket, `PrefixNext` will make the
// high value greater than upper bound, so we store the range here.
r := &ranger.Range{LowVal: []types.Datum{types.NewBytesDatum(low)},
HighVal: []types.Datum{types.NewBytesDatum(high)}}
HighVal: []types.Datum{types.NewBytesDatum(high)}, Collators: collate.GetBinaryCollatorSlice(1)}
feedbackRanges = append(feedbackRanges, r)

if !ran.HighExclude {
Expand Down Expand Up @@ -465,13 +466,15 @@ func SplitRangesAcrossInt64Boundary(ranges []*ranger.Range, keepOrder bool, desc
LowVal: ranges[idx].LowVal,
LowExclude: ranges[idx].LowExclude,
HighVal: []types.Datum{types.NewUintDatum(math.MaxInt64)},
Collators: ranges[idx].Collators,
})
}
if !(ranges[idx].HighVal[0].GetUint64() == math.MaxInt64+1 && ranges[idx].HighExclude) {
unsignedRanges = append(unsignedRanges, &ranger.Range{
LowVal: []types.Datum{types.NewUintDatum(math.MaxInt64 + 1)},
HighVal: ranges[idx].HighVal,
HighExclude: ranges[idx].HighExclude,
Collators: ranges[idx].Collators,
})
}
if idx < len(ranges) {
Expand Down Expand Up @@ -585,7 +588,7 @@ func indexRangesToKVRangesForTablesWithInterruptSignal(sc *stmtctx.StatementCont
return nil, err
}
feedbackRanges = append(feedbackRanges, &ranger.Range{LowVal: []types.Datum{types.NewBytesDatum(low)},
HighVal: []types.Datum{types.NewBytesDatum(high)}, LowExclude: false, HighExclude: true})
HighVal: []types.Datum{types.NewBytesDatum(high)}, LowExclude: false, HighExclude: true, Collators: collate.GetBinaryCollatorSlice(1)})
}
feedbackRanges, ok := fb.Hist.SplitRange(sc, feedbackRanges, true)
if !ok {
Expand Down Expand Up @@ -624,7 +627,7 @@ func CommonHandleRangesToKVRanges(sc *stmtctx.StatementContext, tids []int64, ra
return nil, err
}
rans = append(rans, &ranger.Range{LowVal: []types.Datum{types.NewBytesDatum(low)},
HighVal: []types.Datum{types.NewBytesDatum(high)}, LowExclude: false, HighExclude: true})
HighVal: []types.Datum{types.NewBytesDatum(high)}, LowExclude: false, HighExclude: true, Collators: collate.GetBinaryCollatorSlice(1)})
}
krs := make([]kv.KeyRange, 0, len(rans))
for _, ran := range rans {
Expand Down
47 changes: 35 additions & 12 deletions distsql/request_builder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/codec"
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/ranger"
"github.com/pingcap/tipb/go-tipb"
Expand Down Expand Up @@ -72,29 +73,34 @@ func TestTableHandlesToKVRanges(t *testing.T) {
func TestTableRangesToKVRanges(t *testing.T) {
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(2)},
HighVal: []types.Datum{types.NewIntDatum(4)},
LowExclude: true,
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(4)},
HighVal: []types.Datum{types.NewIntDatum(19)},
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(19)},
HighVal: []types.Datum{types.NewIntDatum(32)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(34)},
HighVal: []types.Datum{types.NewIntDatum(34)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
}

Expand Down Expand Up @@ -129,29 +135,34 @@ func TestTableRangesToKVRanges(t *testing.T) {
func TestIndexRangesToKVRanges(t *testing.T) {
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(2)},
HighVal: []types.Datum{types.NewIntDatum(4)},
LowExclude: true,
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(4)},
HighVal: []types.Datum{types.NewIntDatum(19)},
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(19)},
HighVal: []types.Datum{types.NewIntDatum(32)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(34)},
HighVal: []types.Datum{types.NewIntDatum(34)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
}

Expand Down Expand Up @@ -188,29 +199,34 @@ func TestIndexRangesToKVRanges(t *testing.T) {
func TestRequestBuilder1(t *testing.T) {
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(2)},
HighVal: []types.Datum{types.NewIntDatum(4)},
LowExclude: true,
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(4)},
HighVal: []types.Datum{types.NewIntDatum(19)},
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(19)},
HighVal: []types.Datum{types.NewIntDatum(32)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(34)},
HighVal: []types.Datum{types.NewIntDatum(34)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
}

Expand Down Expand Up @@ -265,29 +281,34 @@ func TestRequestBuilder1(t *testing.T) {
func TestRequestBuilder2(t *testing.T) {
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(2)},
HighVal: []types.Datum{types.NewIntDatum(4)},
LowExclude: true,
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(4)},
HighVal: []types.Datum{types.NewIntDatum(19)},
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(19)},
HighVal: []types.Datum{types.NewIntDatum(32)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(34)},
HighVal: []types.Datum{types.NewIntDatum(34)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
}

Expand Down Expand Up @@ -573,8 +594,9 @@ func TestRequestBuilder8(t *testing.T) {
func TestTableRangesToKVRangesWithFbs(t *testing.T) {
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(4)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(4)},
Collators: collate.GetBinaryCollatorSlice(1),
},
}
fb := newTestFb()
Expand All @@ -594,8 +616,9 @@ func TestTableRangesToKVRangesWithFbs(t *testing.T) {
func TestIndexRangesToKVRangesWithFbs(t *testing.T) {
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(4)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(4)},
Collators: collate.GetBinaryCollatorSlice(1),
},
}
fb := newTestFb()
Expand Down
4 changes: 2 additions & 2 deletions executor/analyze.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2117,7 +2117,7 @@ func analyzeIndexIncremental(idxExec *analyzeIndexIncrementalExec) *statistics.A
if err != nil {
return &statistics.AnalyzeResults{Err: err, Job: idxExec.job}
}
ran := ranger.Range{LowVal: values, HighVal: []types.Datum{types.MaxValueDatum()}}
ran := ranger.Range{LowVal: values, HighVal: []types.Datum{types.MaxValueDatum()}, Collators: collate.GetBinaryCollatorSlice(1)}
hist, cms, fms, topN, err := idxExec.buildStats([]*ranger.Range{&ran}, false)
if err != nil {
return &statistics.AnalyzeResults{Err: err, Job: idxExec.job}
Expand Down Expand Up @@ -2172,7 +2172,7 @@ func analyzePKIncremental(colExec *analyzePKIncrementalExec) *statistics.Analyze
maxVal = types.NewIntDatum(math.MaxInt64)
}
startPos := *colExec.oldHist.GetUpper(colExec.oldHist.Len() - 1)
ran := ranger.Range{LowVal: []types.Datum{startPos}, LowExclude: true, HighVal: []types.Datum{maxVal}}
ran := ranger.Range{LowVal: []types.Datum{startPos}, LowExclude: true, HighVal: []types.Datum{maxVal}, Collators: collate.GetBinaryCollatorSlice(1)}
hists, _, _, _, _, err := colExec.buildStats([]*ranger.Range{&ran}, false)
if err != nil {
return &statistics.AnalyzeResults{Err: err, Job: colExec.job}
Expand Down
2 changes: 2 additions & 0 deletions executor/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4076,6 +4076,7 @@ func buildRangesForIndexJoin(ctx sessionctx.Context, lookUpContents []*indexJoin
ran.HighVal[lastPos] = nextColRan.HighVal[0]
ran.LowExclude = nextColRan.LowExclude
ran.HighExclude = nextColRan.HighExclude
ran.Collators = nextColRan.Collators
tmpDatumRanges = append(tmpDatumRanges, ran.Clone())
}
}
Expand Down Expand Up @@ -4127,6 +4128,7 @@ func buildKvRangesForIndexJoin(ctx sessionctx.Context, tableID, indexID int64, l
ran.HighVal[lastPos] = nextColRan.HighVal[0]
ran.LowExclude = nextColRan.LowExclude
ran.HighExclude = nextColRan.HighExclude
ran.Collators = nextColRan.Collators
tmpDatumRanges = append(tmpDatumRanges, ran.Clone())
}
}
Expand Down
3 changes: 2 additions & 1 deletion executor/executor_pkg_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/mock"
"github.com/pingcap/tidb/util/ranger"
Expand Down Expand Up @@ -212,7 +213,7 @@ func generateIndexRange(vals ...int64) *ranger.Range {
lowDatums := generateDatumSlice(vals...)
highDatums := make([]types.Datum, len(vals))
copy(highDatums, lowDatums)
return &ranger.Range{LowVal: lowDatums, HighVal: highDatums}
return &ranger.Range{LowVal: lowDatums, HighVal: highDatums, Collators: collate.GetBinaryCollatorSlice(len(lowDatums))}
}

func generateDatumSlice(vals ...int64) []types.Datum {
Expand Down
13 changes: 9 additions & 4 deletions planner/core/exhaust_physical_plans.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1538,6 +1538,7 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn
HighVal: make([]types.Datum, pointLength, pointLength+1),
LowExclude: colRan.LowExclude,
HighExclude: colRan.HighExclude,
Collators: make([]collate.Collator, pointLength, pointLength+1),
}
ran.LowVal = append(ran.LowVal, colRan.LowVal[0])
ran.HighVal = append(ran.HighVal, colRan.HighVal[0])
Expand All @@ -1546,13 +1547,15 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn
} else if haveExtraCol {
// Reserve a position for the last col.
ranges = append(ranges, &ranger.Range{
LowVal: make([]types.Datum, pointLength+1),
HighVal: make([]types.Datum, pointLength+1),
LowVal: make([]types.Datum, pointLength+1),
HighVal: make([]types.Datum, pointLength+1),
Collators: make([]collate.Collator, pointLength+1),
})
} else {
ranges = append(ranges, &ranger.Range{
LowVal: make([]types.Datum, pointLength),
HighVal: make([]types.Datum, pointLength),
LowVal: make([]types.Datum, pointLength),
HighVal: make([]types.Datum, pointLength),
Collators: make([]collate.Collator, pointLength),
})
}
sc := ijHelper.join.ctx.GetSessionVars().StmtCtx
Expand All @@ -1575,6 +1578,7 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn
for _, ran := range ranges {
ran.LowVal[i] = oneColumnRan[0].LowVal[0]
ran.HighVal[i] = oneColumnRan[0].HighVal[0]
ran.Collators[i] = oneColumnRan[0].Collators[0]
}
curRangeLen := len(ranges)
for ranIdx := 1; ranIdx < len(oneColumnRan); ranIdx++ {
Expand All @@ -1583,6 +1587,7 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn
newRange := ranges[oldRangeIdx].Clone()
newRange.LowVal[i] = oneColumnRan[ranIdx].LowVal[0]
newRange.HighVal[i] = oneColumnRan[ranIdx].HighVal[0]
newRange.Collators[i] = oneColumnRan[0].Collators[0]
newRanges = append(newRanges, newRange)
}
if sc.MemTracker != nil && len(newRanges) != 0 {
Expand Down
4 changes: 2 additions & 2 deletions planner/core/find_best_task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1520,7 +1520,7 @@ func convertRangeFromExpectedCnt(ranges []*ranger.Range, rangeCounts []float64,
if i < 0 {
return nil, 0, true
}
convertedRanges = []*ranger.Range{{LowVal: ranges[i].HighVal, HighVal: []types.Datum{types.MaxValueDatum()}, LowExclude: !ranges[i].HighExclude}}
convertedRanges = []*ranger.Range{{LowVal: ranges[i].HighVal, HighVal: []types.Datum{types.MaxValueDatum()}, LowExclude: !ranges[i].HighExclude, Collators: ranges[i].Collators}}
} else {
for i = 0; i < len(ranges); i++ {
if count+rangeCounts[i] >= expectedCnt {
Expand All @@ -1531,7 +1531,7 @@ func convertRangeFromExpectedCnt(ranges []*ranger.Range, rangeCounts []float64,
if i == len(ranges) {
return nil, 0, true
}
convertedRanges = []*ranger.Range{{LowVal: []types.Datum{{}}, HighVal: ranges[i].LowVal, HighExclude: !ranges[i].LowExclude}}
convertedRanges = []*ranger.Range{{LowVal: []types.Datum{{}}, HighVal: ranges[i].LowVal, HighExclude: !ranges[i].LowExclude, Collators: ranges[i].Collators}}
}
return convertedRanges, count, false
}
Expand Down
16 changes: 10 additions & 6 deletions planner/util/path_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/planner/util"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/ranger"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -81,16 +82,19 @@ func TestOnlyPointRange(t *testing.T) {
nullDatum := types.MinNotNullDatum()
nullDatum.SetNull()
nullPointRange := ranger.Range{
LowVal: []types.Datum{*nullDatum.Clone()},
HighVal: []types.Datum{*nullDatum.Clone()},
LowVal: []types.Datum{*nullDatum.Clone()},
HighVal: []types.Datum{*nullDatum.Clone()},
Collators: collate.GetBinaryCollatorSlice(1),
}
onePointRange := ranger.Range{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(1)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(1)},
Collators: collate.GetBinaryCollatorSlice(1),
}
one2TwoRange := ranger.Range{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Collators: collate.GetBinaryCollatorSlice(1),
}

intHandlePath := &util.AccessPath{IsIntHandlePath: true}
Expand Down
1 change: 1 addition & 0 deletions statistics/feedback.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,7 @@ func (q *QueryFeedback) DecodeToRanges(isIndex bool) ([]*ranger.Range, error) {
LowVal: lowVal,
HighVal: highVal,
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(len(lowVal)),
}))
}
return ranges, nil
Expand Down
1 change: 1 addition & 0 deletions statistics/handle/handle_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,7 @@ func (s *testStatsSuite) TestColumnIDs(c *C) {
HighVal: []types.Datum{types.NewIntDatum(2)},
LowExclude: false,
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
}
count, err := statsTbl.GetRowCountByColumnRanges(sctx, tableInfo.Columns[0].ID, []*ranger.Range{ran})
c.Assert(err, IsNil)
Expand Down
Loading

0 comments on commit 2fd3b08

Please sign in to comment.