diff --git a/domain/BUILD.bazel b/domain/BUILD.bazel index 6e1618129990e..6e960d7929cbb 100644 --- a/domain/BUILD.bazel +++ b/domain/BUILD.bazel @@ -58,6 +58,7 @@ go_library( "//util/etcd", "//util/execdetails", "//util/expensivequery", + "//util/gctuner", "//util/intest", "//util/logutil", "//util/memory", diff --git a/domain/domain.go b/domain/domain.go index 557a209088672..b1386e95e41a5 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -69,6 +69,7 @@ import ( "github.com/pingcap/tidb/util/engine" "github.com/pingcap/tidb/util/etcd" "github.com/pingcap/tidb/util/expensivequery" + "github.com/pingcap/tidb/util/gctuner" "github.com/pingcap/tidb/util/intest" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" @@ -938,6 +939,7 @@ func (do *Domain) Close() { if do.onClose != nil { do.onClose() } + gctuner.WaitMemoryLimitTunerExitInTest() logutil.BgLogger().Info("domain closed", zap.Duration("take time", time.Since(startTime))) } diff --git a/util/gctuner/BUILD.bazel b/util/gctuner/BUILD.bazel index f097c68c6b761..88cf2efb21590 100644 --- a/util/gctuner/BUILD.bazel +++ b/util/gctuner/BUILD.bazel @@ -12,6 +12,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//util", + "//util/intest", "//util/memory", "@com_github_pingcap_failpoint//:failpoint", "@org_uber_go_atomic//:atomic", diff --git a/util/gctuner/memory_limit_tuner.go b/util/gctuner/memory_limit_tuner.go index 1679e012579d0..204c569c0cd45 100644 --- a/util/gctuner/memory_limit_tuner.go +++ b/util/gctuner/memory_limit_tuner.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/util" + "github.com/pingcap/tidb/util/intest" "github.com/pingcap/tidb/util/memory" atomicutil "go.uber.org/atomic" ) @@ -41,6 +42,17 @@ type memoryLimitTuner struct { // fallbackPercentage indicates the fallback memory limit percentage when turning. const fallbackPercentage float64 = 1.1 +var memoryGoroutineCntInTest = *atomicutil.NewInt64(0) + +// WaitMemoryLimitTunerExitInTest is used to wait memory limit tuner exit in test. +func WaitMemoryLimitTunerExitInTest() { + if intest.InTest { + for memoryGoroutineCntInTest.Load() > 0 { + time.Sleep(100 * time.Millisecond) + } + } +} + // tuning check the memory nextGC and judge whether this GC is trigger by memory limit. // Go runtime ensure that it will be called serially. func (t *memoryLimitTuner) tuning() { @@ -62,10 +74,17 @@ func (t *memoryLimitTuner) tuning() { if float64(r.HeapInuse)*ratio > float64(debug.SetMemoryLimit(-1)) { if t.nextGCTriggeredByMemoryLimit.Load() && t.waitingReset.CompareAndSwap(false, true) { go func() { + if intest.InTest { + memoryGoroutineCntInTest.Inc() + defer memoryGoroutineCntInTest.Dec() + } memory.MemoryLimitGCLast.Store(time.Now()) memory.MemoryLimitGCTotal.Add(1) debug.SetMemoryLimit(t.calcMemoryLimit(fallbackPercentage)) resetInterval := 1 * time.Minute // Wait 1 minute and set back, to avoid frequent GC + if intest.InTest { + resetInterval = 3 * time.Second + } failpoint.Inject("testMemoryLimitTuner", func(val failpoint.Value) { if val, ok := val.(bool); val && ok { resetInterval = 1 * time.Second