From 764d7bf282142ecfdd46ab1baafa881ccb24f066 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Tue, 17 Oct 2023 13:39:02 -0700 Subject: [PATCH 1/5] domain/metrics: add a metric for lease expired time Signed-off-by: Jack Yu --- pkg/domain/schema_validator.go | 1 + pkg/metrics/domain.go | 11 +++++++++++ pkg/metrics/metrics.go | 1 + 3 files changed, 13 insertions(+) diff --git a/pkg/domain/schema_validator.go b/pkg/domain/schema_validator.go index 0372b0aa1a185..8696edd2a5ca4 100644 --- a/pkg/domain/schema_validator.go +++ b/pkg/domain/schema_validator.go @@ -146,6 +146,7 @@ func (s *schemaValidator) Update(leaseGrantTS uint64, oldVer, currVer int64, cha leaseGrantTime := oracle.GetTimeFromTS(leaseGrantTS) leaseExpire := leaseGrantTime.Add(s.lease - time.Millisecond) s.latestSchemaExpire = leaseExpire + metrics.LeaseExpireTime.Set(float64(leaseExpire.Unix())) // Update the schema deltaItem information. if currVer != oldVer { diff --git a/pkg/metrics/domain.go b/pkg/metrics/domain.go index 7a562710a8d87..3a16871a6ae3c 100644 --- a/pkg/metrics/domain.go +++ b/pkg/metrics/domain.go @@ -20,6 +20,9 @@ import ( // Metrics for the domain package. var ( + // LeaseExpireTime records the lease expire time. + LeaseExpireTime prometheus.Gauge + // LoadSchemaCounter records the counter of load schema. LoadSchemaCounter *prometheus.CounterVec @@ -51,6 +54,14 @@ var ( // InitDomainMetrics initializes domain metrics. func InitDomainMetrics() { + LeaseExpireTime = NewGauge( + prometheus.GaugeOpts{ + Namespace: "tidb", + Subsystem: "domain", + Name: "lease_expire_time", + Help: "When the last time the lease is expired", + }) + LoadSchemaCounter = NewCounterVec( prometheus.CounterOpts{ Namespace: "tidb", diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index a429dc130056f..cb97342d35f23 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -158,6 +158,7 @@ func RegisterMetrics() { prometheus.MustRegister(JobsGauge) prometheus.MustRegister(LoadPrivilegeCounter) prometheus.MustRegister(InfoCacheCounters) + prometheus.MustRegister(LeaseExpireTime) prometheus.MustRegister(LoadSchemaCounter) prometheus.MustRegister(LoadSchemaDuration) prometheus.MustRegister(MetaHistogram) From 54e48c38606bc2839a4ca8f9c4b21028f4829b3e Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Tue, 17 Oct 2023 14:04:05 -0700 Subject: [PATCH 2/5] update to microsecond Signed-off-by: Jack Yu --- pkg/domain/schema_validator.go | 2 +- pkg/metrics/domain.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/domain/schema_validator.go b/pkg/domain/schema_validator.go index 8696edd2a5ca4..72c9471dbb2f2 100644 --- a/pkg/domain/schema_validator.go +++ b/pkg/domain/schema_validator.go @@ -146,7 +146,7 @@ func (s *schemaValidator) Update(leaseGrantTS uint64, oldVer, currVer int64, cha leaseGrantTime := oracle.GetTimeFromTS(leaseGrantTS) leaseExpire := leaseGrantTime.Add(s.lease - time.Millisecond) s.latestSchemaExpire = leaseExpire - metrics.LeaseExpireTime.Set(float64(leaseExpire.Unix())) + metrics.LeaseExpireTime.Set(float64(leaseExpire.UnixMicro())) // Update the schema deltaItem information. if currVer != oldVer { diff --git a/pkg/metrics/domain.go b/pkg/metrics/domain.go index 3a16871a6ae3c..9708d4fea769f 100644 --- a/pkg/metrics/domain.go +++ b/pkg/metrics/domain.go @@ -59,7 +59,7 @@ func InitDomainMetrics() { Namespace: "tidb", Subsystem: "domain", Name: "lease_expire_time", - Help: "When the last time the lease is expired", + Help: "When the last time the lease is expired, it is in microseconds", }) LoadSchemaCounter = NewCounterVec( From 292270377036508f2d6c3e2aed3be96587823d48 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Tue, 17 Oct 2023 14:10:38 -0700 Subject: [PATCH 3/5] milliseconds better Signed-off-by: Jack Yu --- pkg/domain/schema_validator.go | 2 +- pkg/metrics/domain.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/domain/schema_validator.go b/pkg/domain/schema_validator.go index 72c9471dbb2f2..0db4f0c7e6f11 100644 --- a/pkg/domain/schema_validator.go +++ b/pkg/domain/schema_validator.go @@ -146,7 +146,7 @@ func (s *schemaValidator) Update(leaseGrantTS uint64, oldVer, currVer int64, cha leaseGrantTime := oracle.GetTimeFromTS(leaseGrantTS) leaseExpire := leaseGrantTime.Add(s.lease - time.Millisecond) s.latestSchemaExpire = leaseExpire - metrics.LeaseExpireTime.Set(float64(leaseExpire.UnixMicro())) + metrics.LeaseExpireTime.Set(float64(leaseExpire.UnixMilli())) // Update the schema deltaItem information. if currVer != oldVer { diff --git a/pkg/metrics/domain.go b/pkg/metrics/domain.go index 9708d4fea769f..c85bb5b114abb 100644 --- a/pkg/metrics/domain.go +++ b/pkg/metrics/domain.go @@ -59,7 +59,7 @@ func InitDomainMetrics() { Namespace: "tidb", Subsystem: "domain", Name: "lease_expire_time", - Help: "When the last time the lease is expired, it is in microseconds", + Help: "When the last time the lease is expired, it is in milliseconds", }) LoadSchemaCounter = NewCounterVec( From 9dfb8fdf3b90175edfc7f93034b5bb1b698b6b20 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Tue, 17 Oct 2023 14:45:34 -0700 Subject: [PATCH 4/5] use seconds Signed-off-by: Jack Yu --- pkg/domain/schema_validator.go | 2 +- pkg/metrics/domain.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/domain/schema_validator.go b/pkg/domain/schema_validator.go index 0db4f0c7e6f11..8696edd2a5ca4 100644 --- a/pkg/domain/schema_validator.go +++ b/pkg/domain/schema_validator.go @@ -146,7 +146,7 @@ func (s *schemaValidator) Update(leaseGrantTS uint64, oldVer, currVer int64, cha leaseGrantTime := oracle.GetTimeFromTS(leaseGrantTS) leaseExpire := leaseGrantTime.Add(s.lease - time.Millisecond) s.latestSchemaExpire = leaseExpire - metrics.LeaseExpireTime.Set(float64(leaseExpire.UnixMilli())) + metrics.LeaseExpireTime.Set(float64(leaseExpire.Unix())) // Update the schema deltaItem information. if currVer != oldVer { diff --git a/pkg/metrics/domain.go b/pkg/metrics/domain.go index c85bb5b114abb..06c131653c36c 100644 --- a/pkg/metrics/domain.go +++ b/pkg/metrics/domain.go @@ -59,7 +59,7 @@ func InitDomainMetrics() { Namespace: "tidb", Subsystem: "domain", Name: "lease_expire_time", - Help: "When the last time the lease is expired, it is in milliseconds", + Help: "When the last time the lease is expired, it is in seconds", }) LoadSchemaCounter = NewCounterVec( From aab60569c6d9d49252ffc1c78ce392fd4993c5aa Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Tue, 17 Oct 2023 15:13:32 -0700 Subject: [PATCH 5/5] update tidb.json Signed-off-by: Jack Yu --- pkg/metrics/grafana/tidb.json | 108 ++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/pkg/metrics/grafana/tidb.json b/pkg/metrics/grafana/tidb.json index 644510fbdf09e..32d1f6d1ab179 100644 --- a/pkg/metrics/grafana/tidb.json +++ b/pkg/metrics/grafana/tidb.json @@ -12901,6 +12901,114 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "How much longer until the lease expires?", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 23763572002, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "tidb_domain_lease_expire_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"} - time()", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "tidb_domain_load_schema_duration_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Lease Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:541", + "format": "dtdurations", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:542", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "repeat": null,