Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

executor: trace the memory usage of Selection executors #13927

Merged
merged 35 commits into from
Dec 23, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
7de1d71
Trace memory usage of Projection Executors
qw4990 Dec 5, 2019
41a7b21
format
qw4990 Dec 5, 2019
95b1ad1
Merge remote-tracking branch 'upstream/master' into mem-proj
qw4990 Dec 5, 2019
fb5fb75
Merge remote-tracking branch 'upstream/master' into mem-proj
qw4990 Dec 5, 2019
05c118c
WIP
qw4990 Dec 5, 2019
c2cde1a
fixCI
qw4990 Dec 5, 2019
8bcf97f
Merge remote-tracking branch 'upstream/master' into mem-proj
qw4990 Dec 5, 2019
553debb
Trace memory usage of Selection Executors
qw4990 Dec 5, 2019
54f032e
fixup
qw4990 Dec 5, 2019
474b747
Merge branch 'mem-proj' into mem-sel
qw4990 Dec 5, 2019
bfb5c71
clean memory when Close
qw4990 Dec 5, 2019
06f9a68
Merge branch 'mem-proj' into mem-sel
qw4990 Dec 5, 2019
9b8f1bc
fixup
qw4990 Dec 5, 2019
85e1e4a
add more tests
qw4990 Dec 9, 2019
1ab7de5
Merge remote-tracking branch 'upstream/master' into mem-proj
qw4990 Dec 9, 2019
ba06f88
Merge branch 'mem-proj' into mem-sel
qw4990 Dec 9, 2019
c25380a
fixup
qw4990 Dec 9, 2019
0d75242
Merge branch 'mem-proj' into mem-sel
qw4990 Dec 9, 2019
70be389
address comment
qw4990 Dec 9, 2019
8038f1d
let main-goroutine wait for all worker-goroutines exit when closing
qw4990 Dec 10, 2019
4304ca3
Merge remote-tracking branch 'upstream/master' into mem-proj
qw4990 Dec 10, 2019
2514307
Merge branch 'mem-proj' into mem-sel
qw4990 Dec 10, 2019
5648fe3
fixup
qw4990 Dec 10, 2019
78df820
Merge branch 'mem-proj' into mem-sel
qw4990 Dec 10, 2019
10fc1ac
update test
qw4990 Dec 10, 2019
21e2f11
update
qw4990 Dec 10, 2019
aa3d7fc
Merge branch 'mem-proj' into mem-sel
qw4990 Dec 10, 2019
b489aae
address comments
qw4990 Dec 10, 2019
3ac6376
Merge branch 'mem-proj' into mem-sel
qw4990 Dec 10, 2019
bd884ee
Merge remote-tracking branch 'upstream/master' into mem-sel
qw4990 Dec 16, 2019
3679891
Merge remote-tracking branch 'upstream/master' into mem-sel
qw4990 Dec 19, 2019
83e3b75
fixup
qw4990 Dec 19, 2019
cf76344
Merge branch 'master' into mem-sel
ichn-hu Dec 20, 2019
5aab369
Merge branch 'master' into mem-sel
qw4990 Dec 23, 2019
d2dd63d
Merge branch 'master' into mem-sel
qw4990 Dec 23, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions executor/executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -1029,14 +1029,19 @@ type SelectionExec struct {
inputIter *chunk.Iterator4Chunk
inputRow chunk.Row
childResult *chunk.Chunk

memTracker *memory.Tracker
}

// Open implements the Executor Open interface.
func (e *SelectionExec) Open(ctx context.Context) error {
if err := e.baseExecutor.Open(ctx); err != nil {
return err
}
e.memTracker = memory.NewTracker(e.id, -1)
e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker)
e.childResult = newFirstChunk(e.children[0])
e.memTracker.Consume(e.childResult.MemoryUsage())
e.batched = expression.Vectorizable(e.filters)
if e.batched {
e.selected = make([]bool, 0, chunk.InitialCapacity)
Expand All @@ -1048,6 +1053,7 @@ func (e *SelectionExec) Open(ctx context.Context) error {

// Close implements plannercore.Plan Close interface.
func (e *SelectionExec) Close() error {
e.memTracker.Consume(-e.childResult.MemoryUsage())
e.childResult = nil
e.selected = nil
return e.baseExecutor.Close()
Expand All @@ -1071,7 +1077,9 @@ func (e *SelectionExec) Next(ctx context.Context, req *chunk.Chunk) error {
}
req.AppendRow(e.inputRow)
}
mSize := e.childResult.MemoryUsage()
err := Next(ctx, e.children[0], e.childResult)
e.memTracker.Consume(e.childResult.MemoryUsage() - mSize)
if err != nil {
return err
}
Expand Down Expand Up @@ -1103,7 +1111,9 @@ func (e *SelectionExec) unBatchedNext(ctx context.Context, chk *chunk.Chunk) err
return nil
}
}
mSize := e.childResult.MemoryUsage()
err := Next(ctx, e.children[0], e.childResult)
e.memTracker.Consume(e.childResult.MemoryUsage() - mSize)
if err != nil {
return err
}
Expand Down
3 changes: 2 additions & 1 deletion executor/explain_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ func (s *testSuite1) TestExplainAnalyzeMemory(c *C) {

func (s *testSuite1) checkMemoryInfo(c *C, tk *testkit.TestKit, sql string) {
memCol := 5
ops := []string{"Join", "Reader", "Top", "Sort", "LookUp", "Projection"}
ops := []string{"Join", "Reader", "Top", "Sort", "LookUp", "Projection", "Selection"}
rows := tk.MustQuery(sql).Rows()
for _, row := range rows {
strs := make([]string, len(row))
Expand Down Expand Up @@ -164,6 +164,7 @@ func (s *testSuite1) TestMemoryUsageAfterClose(c *C) {
tk.MustExec(fmt.Sprintf("insert into t values (%v, %v)", i, i))
}
SQLs := []string{"select v+abs(k) from t",
"select v from t where abs(v) > 0",
"select v from t order by v"}
for _, sql := range SQLs {
tk.MustQuery(sql)
Expand Down