Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

planner: update cost formulas of Index*Join of model2 #39296

Merged
merged 7 commits into from
Nov 23, 2022
Merged

planner: update cost formulas of Index*Join of model2 #39296

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f9cadd4
fixup
qw4990 Nov 21, 2022
5cf49eb
fixup
qw4990 Nov 22, 2022
26b1e0e
fixup
qw4990 Nov 22, 2022
c41fade
fixup
qw4990 Nov 22, 2022
00411d0
Merge remote-tracking branch 'upstream/master' into cost-tpcds
qw4990 Nov 23, 2022
624d796
fixup
qw4990 Nov 23, 2022
3d3e55e
Merge branch 'master' into cost-tpcds
qw4990 Nov 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions executor/explainfor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,12 +462,12 @@ func TestPointGetUserVarPlanCache(t *testing.T) {
tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).Check(testkit.Rows( // can use idx_a
`Projection_9 1.00 root test.t1.a, test.t1.b, test.t2.a, test.t2.b`,
`└─MergeJoin_10 1.00 root inner join, left key:test.t2.a, right key:test.t1.a`,
` ├─Selection_42(Build) 10.00 root eq(test.t1.a, 1)`,
` │ └─TableReader_41 10.00 root data:TableRangeScan_40`,
` │ └─TableRangeScan_40 10.00 cop[tikv] table:t1 range:[1,1], keep order:true, stats:pseudo`,
` └─Selection_39(Probe) 0.80 root not(isnull(test.t2.a))`,
` └─Point_Get_38 1.00 root table:t2, index:idx_a(a) `))
`└─IndexJoin_17 1.00 root inner join, inner:TableReader_13, outer key:test.t2.a, inner key:test.t1.a, equal cond:eq(test.t2.a, test.t1.a)`,
` ├─Selection_44(Build) 0.80 root not(isnull(test.t2.a))`,
` │ └─Point_Get_43 1.00 root table:t2, index:idx_a(a) `,
` └─TableReader_13(Probe) 0.00 root data:Selection_12`,
` └─Selection_12 0.00 cop[tikv] eq(test.t1.a, 1)`,
` └─TableRangeScan_11 0.80 cop[tikv] table:t1 range: decided by [eq(test.t1.a, test.t2.a)], keep order:false, stats:pseudo`))

tk.MustExec("set @a=2")
tk.MustQuery("execute stmt using @a").Check(testkit.Rows(
Expand All @@ -478,12 +478,12 @@ func TestPointGetUserVarPlanCache(t *testing.T) {
tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).Check(testkit.Rows( // can use idx_a
`Projection_9 1.00 root test.t1.a, test.t1.b, test.t2.a, test.t2.b`,
`└─MergeJoin_10 1.00 root inner join, left key:test.t2.a, right key:test.t1.a`,
` ├─Selection_42(Build) 10.00 root eq(test.t1.a, 2)`,
` │ └─TableReader_41 10.00 root data:TableRangeScan_40`,
` │ └─TableRangeScan_40 10.00 cop[tikv] table:t1 range:[2,2], keep order:true, stats:pseudo`,
` └─Selection_39(Probe) 0.80 root not(isnull(test.t2.a))`,
` └─Point_Get_38 1.00 root table:t2, index:idx_a(a) `))
`└─IndexJoin_17 1.00 root inner join, inner:TableReader_13, outer key:test.t2.a, inner key:test.t1.a, equal cond:eq(test.t2.a, test.t1.a)`,
` ├─Selection_44(Build) 0.80 root not(isnull(test.t2.a))`,
` │ └─Point_Get_43 1.00 root table:t2, index:idx_a(a) `,
` └─TableReader_13(Probe) 0.00 root data:Selection_12`,
` └─Selection_12 0.00 cop[tikv] eq(test.t1.a, 2)`,
` └─TableRangeScan_11 0.80 cop[tikv] table:t1 range: decided by [eq(test.t1.a, test.t2.a)], keep order:false, stats:pseudo`))
tk.MustQuery("execute stmt using @a").Check(testkit.Rows(
"2 4 2 2",
))
Expand Down
44 changes: 30 additions & 14 deletions planner/core/plan_cost_ver2.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,57 +534,73 @@ func (p *PhysicalHashJoin) getPlanCostVer2(taskType property.TaskType, option *P
return p.planCostVer2.label(p), nil
}

// getPlanCostVer2 returns the plan-cost of this sub-plan, which is:
// plan-cost = build-child-cost + build-filter-cost +
// (probe-cost + probe-filter-cost) / concurrency
// probe-cost = probe-child-cost * build-rows / batchRatio
func (p *PhysicalIndexJoin) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (costVer2, error) {
func (p *PhysicalIndexJoin) getIndexJoinCostVer2(taskType property.TaskType, option *PlanCostOption, indexJoinType int) (costVer2, error) {
if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) {
return p.planCostVer2, nil
}

build, probe := p.children[1-p.InnerChildIdx], p.children[p.InnerChildIdx]
buildRows := getCardinality(build, option.CostFlag)
buildRowSize := getAvgRowSize(build.Stats(), build.Schema().Columns)
probeRowsOne := getCardinality(probe, option.CostFlag)
probeRowsTot := probeRowsOne * buildRows
probeRowSize := getAvgRowSize(probe.Stats(), probe.Schema().Columns)
buildFilters, probeFilters := p.LeftConditions, p.RightConditions
probeConcurrency := float64(p.ctx.GetSessionVars().IndexLookupJoinConcurrency())
cpuFactor := getTaskCPUFactorVer2(p, taskType)
requestFactor := getTaskRequestFactorVer2(p, taskType)
memFactor := getTaskMemFactorVer2(p, taskType)

buildFilterCost := filterCostVer2(option, buildRows, buildFilters, cpuFactor)
buildChildCost, err := build.getPlanCostVer2(taskType, option)
if err != nil {
return zeroCostVer2, err
}
buildTaskCost := newCostVer2(option, cpuFactor,
buildRows*10*cpuFactor.Value,
"cpu(%v*10*%v)", buildRows, cpuFactor)

probeFilterCost := filterCostVer2(option, probeRowsTot, probeFilters, cpuFactor)
probeChildCost, err := probe.getPlanCostVer2(taskType, option)
if err != nil {
return zeroCostVer2, err
}

var hashTableCost costVer2
switch indexJoinType {
case 1: // IndexHashJoin
hashTableCost = hashBuildCostVer2(option, buildRows, buildRowSize, cols2Exprs(p.RightJoinKeys), cpuFactor, memFactor)
case 2: // IndexMergeJoin
hashTableCost = newZeroCostVer2(traceCost(option))
default: // IndexJoin
hashTableCost = hashBuildCostVer2(option, probeRowsTot, probeRowSize, cols2Exprs(p.LeftJoinKeys), cpuFactor, memFactor)
}

// IndexJoin executes a batch of rows at a time, so the actual cost of this part should be
// `innerCostPerBatch * numberOfBatches` instead of `innerCostPerRow * numberOfOuterRow`.
// Use an empirical value batchRatio to handle this now.
// TODO: remove this empirical value.
batchRatio := 1800.0
batchRatio := 6.0
probeCost := divCostVer2(mulCostVer2(probeChildCost, buildRows), batchRatio)
numTasks := math.Max(buildRows/batchRatio, 0.001)
doubleReadCost := doubleReadCostVer2(option, numTasks, requestFactor)

p.planCostVer2 = sumCostVer2(buildChildCost, buildFilterCost, divCostVer2(sumCostVer2(probeCost, probeFilterCost, doubleReadCost), probeConcurrency))
p.planCostVer2 = sumCostVer2(buildChildCost, buildFilterCost, buildTaskCost, divCostVer2(sumCostVer2(probeCost, probeFilterCost, hashTableCost), probeConcurrency))
p.planCostInit = true
return p.planCostVer2.label(p), nil
}

// getPlanCostVer2 returns the plan-cost of this sub-plan, which is:
// plan-cost = build-child-cost + build-filter-cost +
// (probe-cost + probe-filter-cost) / concurrency
// probe-cost = probe-child-cost * build-rows / batchRatio
func (p *PhysicalIndexJoin) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (costVer2, error) {
return p.getIndexJoinCostVer2(taskType, option, 0)
}

func (p *PhysicalIndexHashJoin) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (costVer2, error) {
// TODO: distinguish IndexHashJoin with IndexJoin
return p.PhysicalIndexJoin.getPlanCostVer2(taskType, option)
return p.getIndexJoinCostVer2(taskType, option, 1)
}

func (p *PhysicalIndexMergeJoin) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (costVer2, error) {
// TODO: distinguish IndexMergeJoin with IndexJoin
return p.PhysicalIndexJoin.getPlanCostVer2(taskType, option)
return p.getIndexJoinCostVer2(taskType, option, 2)
}

// getPlanCostVer2 returns the plan-cost of this sub-plan, which is:
Expand Down
8 changes: 4 additions & 4 deletions planner/core/testdata/integration_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1502,8 +1502,8 @@
{
"SQL": "explain format = 'brief' SELECT t1.pk FROM t1 INNER JOIN t2 ON t1.col1 = t2.pk INNER JOIN t3 ON t1.col3 = t3.pk WHERE t2.col1 IN ('a' , 'b') AND t3.keycol = 'c' AND t1.col2 = 'a' AND t1.col1 != 'abcdef' AND t1.col1 != 'aaaaaa'",
"Plan": [
"IndexJoin 13.81 root inner join, inner:IndexLookUp, outer key:test.t1.col1, inner key:test.t2.pk, equal cond:eq(test.t1.col1, test.t2.pk)",
"├─IndexJoin(Build) 12.50 root inner join, inner:IndexLookUp, outer key:test.t3.pk, inner key:test.t1.col3, equal cond:eq(test.t3.pk, test.t1.col3)",
"IndexHashJoin 13.81 root inner join, inner:IndexLookUp, outer key:test.t1.col1, inner key:test.t2.pk, equal cond:eq(test.t1.col1, test.t2.pk)",
"├─IndexHashJoin(Build) 12.50 root inner join, inner:IndexLookUp, outer key:test.t3.pk, inner key:test.t1.col3, equal cond:eq(test.t3.pk, test.t1.col3)",
"│ ├─IndexLookUp(Build) 10.00 root ",
"│ │ ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t3, index:keycol(keycol, pad1, pad2) range:[\"c\",\"c\"], keep order:false, stats:pseudo",
"│ │ └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t3 keep order:false, stats:pseudo",
Expand All @@ -1522,8 +1522,8 @@
{
"SQL": "explain format = 'brief' SELECT t1.pk FROM t1 LEFT JOIN t2 ON t1.col1 = t2.pk LEFT JOIN t3 ON t1.col3 = t3.pk WHERE t2.col1 IN ('a' , 'b') AND t3.keycol = 'c' AND t1.col2 = 'a' AND t1.col1 != 'abcdef' AND t1.col1 != 'aaaaaa'",
"Plan": [
"IndexJoin 13.81 root inner join, inner:IndexLookUp, outer key:test.t1.col1, inner key:test.t2.pk, equal cond:eq(test.t1.col1, test.t2.pk)",
"├─IndexJoin(Build) 12.50 root inner join, inner:IndexLookUp, outer key:test.t3.pk, inner key:test.t1.col3, equal cond:eq(test.t3.pk, test.t1.col3)",
"IndexHashJoin 13.81 root inner join, inner:IndexLookUp, outer key:test.t1.col1, inner key:test.t2.pk, equal cond:eq(test.t1.col1, test.t2.pk)",
"├─IndexHashJoin(Build) 12.50 root inner join, inner:IndexLookUp, outer key:test.t3.pk, inner key:test.t1.col3, equal cond:eq(test.t3.pk, test.t1.col3)",
"│ ├─IndexLookUp(Build) 10.00 root ",
"│ │ ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t3, index:keycol(keycol, pad1, pad2) range:[\"c\",\"c\"], keep order:false, stats:pseudo",
"│ │ └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t3 keep order:false, stats:pseudo",
Expand Down
Loading