diff --git a/metrics/metrics.go b/metrics/metrics.go index 8c617893e230f..fd1131dfe353e 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -13,7 +13,10 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + tikvmetrics "github.com/pingcap/tidb/store/tikv/metrics" + "github.com/prometheus/client_golang/prometheus" +) var ( // PanicCounter measures the count of panics. @@ -117,21 +120,6 @@ func RegisterMetrics() { prometheus.MustRegister(DbStmtNodeCounter) prometheus.MustRegister(StoreQueryFeedbackCounter) prometheus.MustRegister(GetStoreLimitErrorCounter) - prometheus.MustRegister(TiKVBackoffHistogram) - prometheus.MustRegister(TiKVCoprocessorHistogram) - prometheus.MustRegister(TiKVLoadSafepointCounter) - prometheus.MustRegister(TiKVLockResolverCounter) - prometheus.MustRegister(TiKVRawkvCmdHistogram) - prometheus.MustRegister(TiKVRawkvSizeHistogram) - prometheus.MustRegister(TiKVRegionCacheCounter) - prometheus.MustRegister(TiKVRegionErrorCounter) - prometheus.MustRegister(TiKVSecondaryLockCleanupFailureCounter) - prometheus.MustRegister(TiKVSendReqHistogram) - prometheus.MustRegister(TiKVTxnCmdHistogram) - prometheus.MustRegister(TiKVTxnRegionsNumHistogram) - prometheus.MustRegister(TiKVTxnWriteKVCountHistogram) - prometheus.MustRegister(TiKVTxnWriteSizeHistogram) - prometheus.MustRegister(TiKVLocalLatchWaitTimeHistogram) prometheus.MustRegister(TimeJumpBackCounter) prometheus.MustRegister(TransactionDuration) prometheus.MustRegister(StatementDeadlockDetectDuration) @@ -151,30 +139,15 @@ func RegisterMetrics() { prometheus.MustRegister(TotalQueryProcHistogram) prometheus.MustRegister(TotalCopProcHistogram) prometheus.MustRegister(TotalCopWaitHistogram) - prometheus.MustRegister(TiKVStatusDuration) - prometheus.MustRegister(TiKVStatusCounter) - prometheus.MustRegister(TiKVBatchPendingRequests) - prometheus.MustRegister(TiKVBatchRequests) - prometheus.MustRegister(TiKVBatchWaitDuration) - prometheus.MustRegister(TiKVBatchSendLatency) - prometheus.MustRegister(TiKvBatchWaitOverLoad) - prometheus.MustRegister(TiKVBatchClientUnavailable) - prometheus.MustRegister(TiKVBatchClientWaitEstablish) - prometheus.MustRegister(TiKVRangeTaskStats) - prometheus.MustRegister(TiKVRangeTaskPushDuration) prometheus.MustRegister(HandleSchemaValidate) - prometheus.MustRegister(TiKVTokenWaitDuration) - prometheus.MustRegister(TiKVTxnHeartBeatHistogram) - prometheus.MustRegister(TiKVPessimisticLockKeysDuration) prometheus.MustRegister(GRPCConnTransientFailureCounter) - prometheus.MustRegister(TiKVTTLLifeTimeReachCounter) - prometheus.MustRegister(TiKVNoAvailableConnectionCounter) - prometheus.MustRegister(TiKVAsyncCommitTxnCounter) - prometheus.MustRegister(TiKVOnePCTxnCounter) prometheus.MustRegister(MaxProcs) prometheus.MustRegister(GOGC) prometheus.MustRegister(ConnIdleDurationHistogram) prometheus.MustRegister(ServerInfo) prometheus.MustRegister(TokenGauge) prometheus.MustRegister(ConfigStatus) + + tikvmetrics.InitMetrics("tidb", "tikvclient") + tikvmetrics.RegisterMetrics() } diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go index 369ea0fd419aa..44cdbbca49591 100644 --- a/store/tikv/2pc.go +++ b/store/tikv/2pc.go @@ -34,9 +34,9 @@ import ( "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/sessionctx/binloginfo" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/table/tables" diff --git a/store/tikv/backoff.go b/store/tikv/backoff.go index 3609f50b029ed..f11f432443ccd 100644 --- a/store/tikv/backoff.go +++ b/store/tikv/backoff.go @@ -26,8 +26,8 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/util/execdetails" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" diff --git a/store/tikv/cleanup.go b/store/tikv/cleanup.go index a0b48b09dc575..5819895a83b55 100644 --- a/store/tikv/cleanup.go +++ b/store/tikv/cleanup.go @@ -16,8 +16,8 @@ package tikv import ( "github.com/pingcap/errors" pb "github.com/pingcap/kvproto/pkg/kvrpcpb" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" diff --git a/store/tikv/client.go b/store/tikv/client.go index 9f03550d280ef..d08f06e26d53a 100644 --- a/store/tikv/client.go +++ b/store/tikv/client.go @@ -35,9 +35,10 @@ import ( "github.com/pingcap/parser/terror" tidbcfg "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" + tidbmetrics "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/config" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/util/execdetails" "github.com/prometheus/client_golang/prometheus" @@ -374,7 +375,7 @@ func (c *rpcClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R clientConn := connArray.Get() if state := clientConn.GetState(); state == connectivity.TransientFailure { storeID := strconv.FormatUint(req.Context.GetPeer().GetStoreId(), 10) - metrics.GRPCConnTransientFailureCounter.WithLabelValues(addr, storeID).Inc() + tidbmetrics.GRPCConnTransientFailureCounter.WithLabelValues(addr, storeID).Inc() } if req.IsDebugReq() { diff --git a/store/tikv/client_batch.go b/store/tikv/client_batch.go index 5117dcd19a798..019344cf3d3dc 100644 --- a/store/tikv/client_batch.go +++ b/store/tikv/client_batch.go @@ -26,9 +26,10 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/tikvpb" "github.com/pingcap/parser/terror" - "github.com/pingcap/tidb/metrics" + tidbmetrics "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/config" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -248,7 +249,7 @@ func (c *batchCommandsClient) send(request *tikvpb.BatchCommandsRequest, entries func (c *batchCommandsClient) recv() (resp *tikvpb.BatchCommandsResponse, err error) { defer func() { if r := recover(); r != nil { - metrics.PanicCounter.WithLabelValues(metrics.LabelBatchRecvLoop).Inc() + tidbmetrics.PanicCounter.WithLabelValues(metrics.LabelBatchRecvLoop).Inc() logutil.BgLogger().Error("batchCommandsClient.recv panic", zap.Reflect("r", r), zap.Stack("stack")) @@ -330,7 +331,7 @@ func (c *batchCommandsClient) reCreateStreamingClientOnce(perr error) error { func (c *batchCommandsClient) batchRecvLoop(cfg config.TiKVClient, tikvTransportLayerLoad *uint64) { defer func() { if r := recover(); r != nil { - metrics.PanicCounter.WithLabelValues(metrics.LabelBatchRecvLoop).Inc() + tidbmetrics.PanicCounter.WithLabelValues(metrics.LabelBatchRecvLoop).Inc() logutil.BgLogger().Error("batchRecvLoop", zap.Reflect("r", r), zap.Stack("stack")) @@ -447,7 +448,7 @@ func resetRequests(requests []*tikvpb.BatchCommandsRequest_Request) []*tikvpb.Ba func (a *batchConn) batchSendLoop(cfg config.TiKVClient) { defer func() { if r := recover(); r != nil { - metrics.PanicCounter.WithLabelValues(metrics.LabelBatchSendLoop).Inc() + tidbmetrics.PanicCounter.WithLabelValues(metrics.LabelBatchSendLoop).Inc() logutil.BgLogger().Error("batchSendLoop", zap.Reflect("r", r), zap.Stack("stack")) diff --git a/store/tikv/commit.go b/store/tikv/commit.go index cf39f11ea7150..b521cb1493538 100644 --- a/store/tikv/commit.go +++ b/store/tikv/commit.go @@ -20,8 +20,8 @@ import ( "github.com/pingcap/errors" pb "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" diff --git a/store/tikv/coprocessor.go b/store/tikv/coprocessor.go index f45ec0cd5d793..72f81ce3ee8ed 100644 --- a/store/tikv/coprocessor.go +++ b/store/tikv/coprocessor.go @@ -36,8 +36,9 @@ import ( "github.com/pingcap/tidb/domain/infosync" "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" + tidbmetrics "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/memory" @@ -48,7 +49,7 @@ import ( var ( tikvTxnRegionsNumHistogramWithCoprocessor = metrics.TiKVTxnRegionsNumHistogram.WithLabelValues("coprocessor") tikvTxnRegionsNumHistogramWithBatchCoprocessor = metrics.TiKVTxnRegionsNumHistogram.WithLabelValues("batch_coprocessor") - coprCacheHistogramEvict = metrics.DistSQLCoprCacheHistogram.WithLabelValues("evict") + coprCacheHistogramEvict = tidbmetrics.DistSQLCoprCacheHistogram.WithLabelValues("evict") ) // CopClient is coprocessor client. diff --git a/store/tikv/kv.go b/store/tikv/kv.go index 1c31b694d79fe..fde4e84a50cfa 100644 --- a/store/tikv/kv.go +++ b/store/tikv/kv.go @@ -29,10 +29,10 @@ import ( "github.com/pingcap/failpoint" tidbcfg "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/config" "github.com/pingcap/tidb/store/tikv/latch" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/oracle/oracles" "github.com/pingcap/tidb/store/tikv/tikvrpc" diff --git a/store/tikv/lock_resolver.go b/store/tikv/lock_resolver.go index 89f7d0d1331ef..079ac0733cadb 100644 --- a/store/tikv/lock_resolver.go +++ b/store/tikv/lock_resolver.go @@ -26,9 +26,9 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/config" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/util/execdetails" diff --git a/metrics/tikvclient.go b/store/tikv/metrics/metrics.go similarity index 57% rename from metrics/tikvclient.go rename to store/tikv/metrics/metrics.go index fc4cf0afb962d..44e726bd0a4d6 100644 --- a/metrics/tikvclient.go +++ b/store/tikv/metrics/metrics.go @@ -1,4 +1,4 @@ -// Copyright 2018 PingCAP, Inc. +// Copyright 2021 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,12 +15,63 @@ package metrics import "github.com/prometheus/client_golang/prometheus" -// TiKVClient metrics. +// Client metrics. var ( + TiKVTxnCmdHistogram *prometheus.HistogramVec + TiKVBackoffHistogram *prometheus.HistogramVec + TiKVSendReqHistogram *prometheus.HistogramVec + TiKVCoprocessorHistogram prometheus.Histogram + TiKVLockResolverCounter *prometheus.CounterVec + TiKVRegionErrorCounter *prometheus.CounterVec + TiKVTxnWriteKVCountHistogram prometheus.Histogram + TiKVTxnWriteSizeHistogram prometheus.Histogram + TiKVRawkvCmdHistogram *prometheus.HistogramVec + TiKVRawkvSizeHistogram *prometheus.HistogramVec + TiKVTxnRegionsNumHistogram *prometheus.HistogramVec + TiKVLoadSafepointCounter *prometheus.CounterVec + TiKVSecondaryLockCleanupFailureCounter *prometheus.CounterVec + TiKVRegionCacheCounter *prometheus.CounterVec + TiKVLocalLatchWaitTimeHistogram prometheus.Histogram + TiKVStatusDuration *prometheus.HistogramVec + TiKVStatusCounter *prometheus.CounterVec + TiKVBatchWaitDuration prometheus.Histogram + TiKVBatchSendLatency prometheus.Histogram + TiKvBatchWaitOverLoad prometheus.Counter + TiKVBatchPendingRequests *prometheus.HistogramVec + TiKVBatchRequests *prometheus.HistogramVec + TiKVBatchClientUnavailable prometheus.Histogram + TiKVBatchClientWaitEstablish prometheus.Histogram + TiKVRangeTaskStats *prometheus.GaugeVec + TiKVRangeTaskPushDuration *prometheus.HistogramVec + TiKVTokenWaitDuration prometheus.Histogram + TiKVTxnHeartBeatHistogram *prometheus.HistogramVec + TiKVPessimisticLockKeysDuration prometheus.Histogram + TiKVTTLLifeTimeReachCounter prometheus.Counter + TiKVNoAvailableConnectionCounter prometheus.Counter + TiKVAsyncCommitTxnCounter *prometheus.CounterVec + TiKVOnePCTxnCounter *prometheus.CounterVec +) + +// Label constants. +const ( + LblType = "type" + LblResult = "result" + LblStore = "store" + LblCommit = "commit" + LblAbort = "abort" + LblRollback = "rollback" + LblBatchGet = "batch_get" + LblGet = "get" + LblLockKeys = "lock_keys" + LabelBatchRecvLoop = "batch-recv-loop" + LabelBatchSendLoop = "batch-send-loop" +) + +func initMetrics(namespace, subsystem string) { TiKVTxnCmdHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "txn_cmd_duration_seconds", Help: "Bucketed histogram of processing time of txn cmds.", Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days @@ -28,8 +79,8 @@ var ( TiKVBackoffHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "backoff_seconds", Help: "total backoff seconds of a single backoffer.", Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days @@ -37,8 +88,8 @@ var ( TiKVSendReqHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "request_seconds", Help: "Bucketed histogram of sending request duration.", Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days @@ -46,8 +97,8 @@ var ( TiKVCoprocessorHistogram = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "cop_duration_seconds", Help: "Run duration of a single coprocessor task, includes backoff time.", Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days @@ -55,24 +106,24 @@ var ( TiKVLockResolverCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "lock_resolver_actions_total", Help: "Counter of lock resolver actions.", }, []string{LblType}) TiKVRegionErrorCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "region_err_total", Help: "Counter of region errors.", }, []string{LblType}) TiKVTxnWriteKVCountHistogram = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "txn_write_kv_num", Help: "Count of kv pairs to write in a transaction.", Buckets: prometheus.ExponentialBuckets(1, 4, 17), // 1 ~ 4G @@ -80,8 +131,8 @@ var ( TiKVTxnWriteSizeHistogram = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "txn_write_size_bytes", Help: "Size of kv pairs to write in a transaction.", Buckets: prometheus.ExponentialBuckets(16, 4, 17), // 16Bytes ~ 64GB @@ -89,8 +140,8 @@ var ( TiKVRawkvCmdHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "rawkv_cmd_seconds", Help: "Bucketed histogram of processing time of rawkv cmds.", Buckets: prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days @@ -98,8 +149,8 @@ var ( TiKVRawkvSizeHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "rawkv_kv_size_bytes", Help: "Size of key/value to put, in bytes.", Buckets: prometheus.ExponentialBuckets(1, 2, 30), // 1Byte ~ 512MB @@ -107,8 +158,8 @@ var ( TiKVTxnRegionsNumHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "txn_regions_num", Help: "Number of regions in a transaction.", Buckets: prometheus.ExponentialBuckets(1, 2, 25), // 1 ~ 16M @@ -116,32 +167,32 @@ var ( TiKVLoadSafepointCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "load_safepoint_total", Help: "Counter of load safepoint.", }, []string{LblType}) TiKVSecondaryLockCleanupFailureCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "lock_cleanup_task_total", Help: "failure statistic of secondary lock cleanup task.", }, []string{LblType}) TiKVRegionCacheCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "region_cache_operations_total", Help: "Counter of region cache.", }, []string{LblType, LblResult}) TiKVLocalLatchWaitTimeHistogram = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "local_latch_wait_seconds", Help: "Wait time of a get local latch.", Buckets: prometheus.ExponentialBuckets(0.0005, 2, 20), // 0.5ms ~ 262s @@ -149,8 +200,8 @@ var ( TiKVStatusDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "kv_status_api_duration", Help: "duration for kv status api.", Buckets: prometheus.ExponentialBuckets(0.0005, 2, 20), // 0.5ms ~ 262s @@ -158,63 +209,63 @@ var ( TiKVStatusCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "kv_status_api_count", Help: "Counter of access kv status api.", }, []string{LblResult}) TiKVBatchWaitDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "batch_wait_duration", Buckets: prometheus.ExponentialBuckets(1, 2, 34), // 1ns ~ 8s Help: "batch wait duration", }) TiKVBatchSendLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "batch_send_latency", Buckets: prometheus.ExponentialBuckets(1, 2, 34), // 1ns ~ 8s Help: "batch send latency", }) TiKvBatchWaitOverLoad = prometheus.NewCounter( prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "batch_wait_overload", Help: "event of tikv transport layer overload", }) TiKVBatchPendingRequests = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "batch_pending_requests", Buckets: prometheus.ExponentialBuckets(1, 2, 8), Help: "number of requests pending in the batch channel", }, []string{"store"}) TiKVBatchRequests = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "batch_requests", Buckets: prometheus.ExponentialBuckets(1, 2, 8), Help: "number of requests in one batch", }, []string{"store"}) TiKVBatchClientUnavailable = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "batch_client_unavailable_seconds", Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days Help: "batch client unavailable", }) TiKVBatchClientWaitEstablish = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "batch_client_wait_connection_establish", Buckets: prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days Help: "batch client wait new connection establish", @@ -222,24 +273,24 @@ var ( TiKVRangeTaskStats = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "range_task_stats", Help: "stat of range tasks", }, []string{LblType, LblResult}) TiKVRangeTaskPushDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "range_task_push_duration", Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 524s Help: "duration to push sub tasks to range task workers", }, []string{LblType}) TiKVTokenWaitDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "batch_executor_token_wait_duration", Buckets: prometheus.ExponentialBuckets(1, 2, 34), // 1ns ~ 8s Help: "tidb txn token wait duration to process batches", @@ -247,16 +298,16 @@ var ( TiKVTxnHeartBeatHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "txn_heart_beat", Help: "Bucketed histogram of the txn_heartbeat request duration.", Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 524s }, []string{LblType}) TiKVPessimisticLockKeysDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "pessimistic_lock_keys_duration", Buckets: prometheus.ExponentialBuckets(0.001, 2, 24), // 1ms ~ 8389s Help: "tidb txn pessimistic lock keys duration", @@ -264,33 +315,80 @@ var ( TiKVTTLLifeTimeReachCounter = prometheus.NewCounter( prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "ttl_lifetime_reach_total", Help: "Counter of ttlManager live too long.", }) TiKVNoAvailableConnectionCounter = prometheus.NewCounter( prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "batch_client_no_available_connection_total", Help: "Counter of no available batch client.", }) TiKVAsyncCommitTxnCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "async_commit_txn_counter", Help: "Counter of async commit transactions.", }, []string{LblType}) TiKVOnePCTxnCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "tikvclient", + Namespace: namespace, + Subsystem: subsystem, Name: "one_pc_txn_counter", Help: "Counter of 1PC transactions.", }, []string{LblType}) -) +} + +func init() { + initMetrics("tikv", "client_go") +} + +// InitMetrics initializes metrics variables with given namespace and subsystem name. +func InitMetrics(namespace, subsystem string) { + initMetrics(namespace, subsystem) +} + +// RegisterMetrics registers all metrics variables. +// Note: to change default namespace and subsystem name, call `InitMetrics` before registering. +func RegisterMetrics() { + prometheus.MustRegister(TiKVTxnCmdHistogram) + prometheus.MustRegister(TiKVBackoffHistogram) + prometheus.MustRegister(TiKVSendReqHistogram) + prometheus.MustRegister(TiKVCoprocessorHistogram) + prometheus.MustRegister(TiKVLockResolverCounter) + prometheus.MustRegister(TiKVRegionErrorCounter) + prometheus.MustRegister(TiKVTxnWriteKVCountHistogram) + prometheus.MustRegister(TiKVTxnWriteSizeHistogram) + prometheus.MustRegister(TiKVRawkvCmdHistogram) + prometheus.MustRegister(TiKVRawkvSizeHistogram) + prometheus.MustRegister(TiKVTxnRegionsNumHistogram) + prometheus.MustRegister(TiKVLoadSafepointCounter) + prometheus.MustRegister(TiKVSecondaryLockCleanupFailureCounter) + prometheus.MustRegister(TiKVRegionCacheCounter) + prometheus.MustRegister(TiKVLocalLatchWaitTimeHistogram) + prometheus.MustRegister(TiKVStatusDuration) + prometheus.MustRegister(TiKVStatusCounter) + prometheus.MustRegister(TiKVBatchWaitDuration) + prometheus.MustRegister(TiKVBatchSendLatency) + prometheus.MustRegister(TiKvBatchWaitOverLoad) + prometheus.MustRegister(TiKVBatchPendingRequests) + prometheus.MustRegister(TiKVBatchRequests) + prometheus.MustRegister(TiKVBatchClientUnavailable) + prometheus.MustRegister(TiKVBatchClientWaitEstablish) + prometheus.MustRegister(TiKVRangeTaskStats) + prometheus.MustRegister(TiKVRangeTaskPushDuration) + prometheus.MustRegister(TiKVTokenWaitDuration) + prometheus.MustRegister(TiKVTxnHeartBeatHistogram) + prometheus.MustRegister(TiKVPessimisticLockKeysDuration) + prometheus.MustRegister(TiKVTTLLifeTimeReachCounter) + prometheus.MustRegister(TiKVNoAvailableConnectionCounter) + prometheus.MustRegister(TiKVAsyncCommitTxnCounter) + prometheus.MustRegister(TiKVOnePCTxnCounter) +} diff --git a/store/tikv/pessimistic.go b/store/tikv/pessimistic.go index 45605e2ae2ec0..ea17711fc4edc 100644 --- a/store/tikv/pessimistic.go +++ b/store/tikv/pessimistic.go @@ -24,8 +24,8 @@ import ( "github.com/pingcap/failpoint" pb "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" diff --git a/store/tikv/prewrite.go b/store/tikv/prewrite.go index 716c2a99b1393..e4cbc6c7eabe3 100644 --- a/store/tikv/prewrite.go +++ b/store/tikv/prewrite.go @@ -24,8 +24,8 @@ import ( "github.com/pingcap/failpoint" pb "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/config" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" diff --git a/store/tikv/range_task.go b/store/tikv/range_task.go index c08f741b9c4dd..57960c21f126b 100644 --- a/store/tikv/range_task.go +++ b/store/tikv/range_task.go @@ -22,8 +22,8 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "go.uber.org/zap" ) diff --git a/store/tikv/rawkv.go b/store/tikv/rawkv.go index a69322a83210d..5542df6ec8738 100644 --- a/store/tikv/rawkv.go +++ b/store/tikv/rawkv.go @@ -20,10 +20,10 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/config" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" - "github.com/tikv/pd/client" + pd "github.com/tikv/pd/client" ) var ( diff --git a/store/tikv/region_cache.go b/store/tikv/region_cache.go index e150515a4e428..99028a58161b9 100644 --- a/store/tikv/region_cache.go +++ b/store/tikv/region_cache.go @@ -32,8 +32,8 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl/placement" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/util" pd "github.com/tikv/pd/client" atomic2 "go.uber.org/atomic" diff --git a/store/tikv/region_request.go b/store/tikv/region_request.go index efd33276f25af..8fc0dd3217f38 100644 --- a/store/tikv/region_request.go +++ b/store/tikv/region_request.go @@ -33,9 +33,10 @@ import ( "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" + tidbmetrics "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/storeutil" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/util/execdetails" @@ -532,7 +533,7 @@ func (s *RegionRequestSender) getStoreToken(st *Store, limit int64) error { st.tokenCount.Add(1) return nil } - metrics.GetStoreLimitErrorCounter.WithLabelValues(st.addr, strconv.FormatUint(st.storeID, 10)).Inc() + tidbmetrics.GetStoreLimitErrorCounter.WithLabelValues(st.addr, strconv.FormatUint(st.storeID, 10)).Inc() return ErrTokenLimit.GenWithStackByArgs(st.storeID) } diff --git a/store/tikv/snapshot.go b/store/tikv/snapshot.go index 306319a413259..28bbabd1b2f20 100644 --- a/store/tikv/snapshot.go +++ b/store/tikv/snapshot.go @@ -30,8 +30,8 @@ import ( "github.com/pingcap/failpoint" pb "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/execdetails" diff --git a/store/tikv/txn.go b/store/tikv/txn.go index 32d2b80a88720..53de1fcc52ec9 100644 --- a/store/tikv/txn.go +++ b/store/tikv/txn.go @@ -30,9 +30,9 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/tikv/logutil" + "github.com/pingcap/tidb/store/tikv/metrics" "github.com/pingcap/tidb/util/execdetails" "go.uber.org/zap" )