From 2350b03e5f085af23dd119e10d82a965990ab941 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 6 Feb 2023 23:21:53 +0800 Subject: [PATCH 1/5] metrics: add tidb_rm_pool_concurrency Signed-off-by: Weizhen Wang --- metrics/resourcemanager.go | 7 +++++++ util/gpool/spmc/BUILD.bazel | 2 ++ util/gpool/spmc/spmcpool.go | 31 ++++++++++++++++++------------- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/metrics/resourcemanager.go b/metrics/resourcemanager.go index d45b8833e1225..989a420bd5213 100644 --- a/metrics/resourcemanager.go +++ b/metrics/resourcemanager.go @@ -24,4 +24,11 @@ var ( Name: "ema_cpu_usage", Help: "exponential moving average of CPU usage", }) + PoolConcurrencyCounter = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "tidb", + Subsystem: "rm", + Name: "pool_concurrency", + Help: "How many times transactions enter this state", + }, []string{LblType}) ) diff --git a/util/gpool/spmc/BUILD.bazel b/util/gpool/spmc/BUILD.bazel index db4d724052666..68028a5573b34 100644 --- a/util/gpool/spmc/BUILD.bazel +++ b/util/gpool/spmc/BUILD.bazel @@ -11,6 +11,7 @@ go_library( importpath = "github.com/pingcap/tidb/util/gpool/spmc", visibility = ["//visibility:public"], deps = [ + "//metrics", "//resourcemanager", "//resourcemanager/pooltask", "//resourcemanager/util", @@ -18,6 +19,7 @@ go_library( "//util/logutil", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_log//:log", + "@com_github_prometheus_client_golang//prometheus", "@org_uber_go_atomic//:atomic", "@org_uber_go_zap//:zap", ], diff --git a/util/gpool/spmc/spmcpool.go b/util/gpool/spmc/spmcpool.go index 35b258957fd62..c42a680271e28 100644 --- a/util/gpool/spmc/spmcpool.go +++ b/util/gpool/spmc/spmcpool.go @@ -21,11 +21,13 @@ import ( "time" "github.com/pingcap/log" + "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/resourcemanager" "github.com/pingcap/tidb/resourcemanager/pooltask" "github.com/pingcap/tidb/resourcemanager/util" "github.com/pingcap/tidb/util/gpool" "github.com/pingcap/tidb/util/logutil" + "github.com/prometheus/client_golang/prometheus" atomicutil "go.uber.org/atomic" "go.uber.org/zap" ) @@ -47,13 +49,14 @@ type Pool[T any, U any, C any, CT any, TF pooltask.Context[CT]] struct { workers *loopQueue[T, U, C, CT, TF] options *Options gpool.BasePool - taskManager pooltask.TaskManager[T, U, C, CT, TF] - waitingTask atomicutil.Uint32 - capacity atomic.Int32 - running atomic.Int32 - state atomic.Int32 - waiting atomic.Int32 // waiting is the number of goroutines that are waiting for the pool to be available. - heartbeatDone atomic.Bool + taskManager pooltask.TaskManager[T, U, C, CT, TF] + waitingTask atomicutil.Uint32 + capacity atomic.Int32 + running atomic.Int32 + state atomic.Int32 + waiting atomic.Int32 // waiting is the number of goroutines that are waiting for the pool to be available. + heartbeatDone atomic.Bool + concurrencyMetrics prometheus.Gauge } // NewSPMCPool create a single producer, multiple consumer goroutine pool. @@ -63,12 +66,13 @@ func NewSPMCPool[T any, U any, C any, CT any, TF pooltask.Context[CT]](name stri opts.ExpiryDuration = gpool.DefaultCleanIntervalTime } result := &Pool[T, U, C, CT, TF]{ - BasePool: gpool.NewBasePool(), - taskCh: make(chan *pooltask.TaskBox[T, U, C, CT, TF], 128), - stopCh: make(chan struct{}), - lock: gpool.NewSpinLock(), - taskManager: pooltask.NewTaskManager[T, U, C, CT, TF](size), - options: opts, + BasePool: gpool.NewBasePool(), + taskCh: make(chan *pooltask.TaskBox[T, U, C, CT, TF], 128), + stopCh: make(chan struct{}), + lock: gpool.NewSpinLock(), + taskManager: pooltask.NewTaskManager[T, U, C, CT, TF](size), + concurrencyMetrics: metrics.PoolConcurrencyCounter.WithLabelValues(name), + options: opts, } result.SetName(name) result.state.Store(int32(gpool.OPENED)) @@ -141,6 +145,7 @@ func (p *Pool[T, U, C, CT, TF]) Tune(size int) { } p.SetLastTuneTs(time.Now()) p.capacity.Store(int32(size)) + p.concurrencyMetrics.Set(float64(size)) if size > capacity { for i := 0; i < size-capacity; i++ { if tid, boostTask := p.taskManager.Overclock(size); boostTask != nil { From bd9b366374815cad2d353d52bb0cc26880705e28 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 6 Feb 2023 23:28:42 +0800 Subject: [PATCH 2/5] metrics: add tidb_rm_pool_concurrency Signed-off-by: Weizhen Wang --- metrics/metrics.go | 1 + metrics/resourcemanager.go | 1 + 2 files changed, 2 insertions(+) diff --git a/metrics/metrics.go b/metrics/metrics.go index f8bbf11b6ef6d..6767d2dfc6adb 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -220,6 +220,7 @@ func RegisterMetrics() { prometheus.MustRegister(TTLPhaseTime) prometheus.MustRegister(EMACPUUsageGauge) + prometheus.MustRegister(PoolConcurrencyCounter) prometheus.MustRegister(HistoricalStatsCounter) prometheus.MustRegister(PlanReplayerTaskCounter) diff --git a/metrics/resourcemanager.go b/metrics/resourcemanager.go index 989a420bd5213..05e4ef0e6e8f4 100644 --- a/metrics/resourcemanager.go +++ b/metrics/resourcemanager.go @@ -24,6 +24,7 @@ var ( Name: "ema_cpu_usage", Help: "exponential moving average of CPU usage", }) + // PoolConcurrencyCounter means how much concurrency in the pool PoolConcurrencyCounter = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "tidb", From 34d0aad9ba3b0d92cd6f9d51bd515542d5e87f19 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 7 Feb 2023 10:39:19 +0800 Subject: [PATCH 3/5] update Signed-off-by: Weizhen Wang --- util/gpool/spmc/spmcpool.go | 1 + 1 file changed, 1 insertion(+) diff --git a/util/gpool/spmc/spmcpool.go b/util/gpool/spmc/spmcpool.go index c42a680271e28..d1a3bbb80c4aa 100644 --- a/util/gpool/spmc/spmcpool.go +++ b/util/gpool/spmc/spmcpool.go @@ -82,6 +82,7 @@ func NewSPMCPool[T any, U any, C any, CT any, TF pooltask.Context[CT]](name stri } } result.capacity.Add(size) + result.concurrencyMetrics.Set(float64(size)) result.workers = newWorkerLoopQueue[T, U, C, CT, TF](int(size)) result.cond = sync.NewCond(result.lock) err := resourcemanager.InstanceResourceManager.Register(result, name, component) From d6f6df474d5103ffedfd7e63455d9d32c5a96940 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 7 Feb 2023 10:48:23 +0800 Subject: [PATCH 4/5] update Signed-off-by: Weizhen Wang --- metrics/resourcemanager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/resourcemanager.go b/metrics/resourcemanager.go index 05e4ef0e6e8f4..c6715512500d4 100644 --- a/metrics/resourcemanager.go +++ b/metrics/resourcemanager.go @@ -30,6 +30,6 @@ var ( Namespace: "tidb", Subsystem: "rm", Name: "pool_concurrency", - Help: "How many times transactions enter this state", + Help: "How many concurrency in the pool", }, []string{LblType}) ) From 2e013f1a815c9213a019f92215de121da679e56a Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 7 Feb 2023 10:53:28 +0800 Subject: [PATCH 5/5] update Signed-off-by: Weizhen Wang --- .bazelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 11c7c8f9ad2ba..6d31d4e95ac74 100644 --- a/.bazelrc +++ b/.bazelrc @@ -20,7 +20,7 @@ build:race --@io_bazel_rules_go//go/config:race --test_env=GORACE=halt_on_error= test --test_env=TZ=Asia/Shanghai test --test_output=errors --test_summary=testcase -test:ci --color=yes +test:ci --color=yes --spawn_strategy=local test:ci --verbose_failures --test_verbose_timeout_warnings test:ci --test_env=GO_TEST_WRAP_TESTV=1 test:ci --experimental_ui_max_stdouterr_bytes=104857600