From 74df4d9ad158c52901e886ed3dd82ab07932ac44 Mon Sep 17 00:00:00 2001
From: Jack Yu <jackysp@gmail.com>
Date: Mon, 6 Nov 2023 17:03:08 -0800
Subject: [PATCH] domain/metrics: add a metric for lease expired time (#47730)
 (#47819)

close pingcap/tidb#47729
---
 domain/schema_validator.go |   1 +
 metrics/domain.go          |   8 +++
 metrics/grafana/tidb.json  | 108 +++++++++++++++++++++++++++++++++++++
 metrics/metrics.go         |   1 +
 4 files changed, 118 insertions(+)

diff --git a/domain/schema_validator.go b/domain/schema_validator.go
index 6f028fdca2e70..9b1d0047fab5c 100644
--- a/domain/schema_validator.go
+++ b/domain/schema_validator.go
@@ -138,6 +138,7 @@ func (s *schemaValidator) Update(leaseGrantTS uint64, oldVer, currVer int64, cha
 	leaseGrantTime := oracle.GetTimeFromTS(leaseGrantTS)
 	leaseExpire := leaseGrantTime.Add(s.lease - time.Millisecond)
 	s.latestSchemaExpire = leaseExpire
+	metrics.LeaseExpireTime.Set(float64(leaseExpire.Unix()))
 
 	// Update the schema deltaItem information.
 	if currVer != oldVer {
diff --git a/metrics/domain.go b/metrics/domain.go
index a76062a811991..f97978c09996c 100644
--- a/metrics/domain.go
+++ b/metrics/domain.go
@@ -21,6 +21,14 @@ import (
 // Metrics for the domain package.
 var (
 	// LoadSchemaCounter records the counter of load schema.
+	LeaseExpireTime = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: "tidb",
+			Subsystem: "domain",
+			Name:      "lease_expire_time",
+			Help:      "When the last time the lease is expired, it is in seconds",
+		})
+
 	LoadSchemaCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: "tidb",
diff --git a/metrics/grafana/tidb.json b/metrics/grafana/tidb.json
index 6ee35f83bbb85..7c9501a269b33 100644
--- a/metrics/grafana/tidb.json
+++ b/metrics/grafana/tidb.json
@@ -12187,6 +12187,114 @@
             "align": false,
             "alignLevel": null
           }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "How much longer until the lease expires?",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {},
+            "overrides": []
+          },
+          "fill": 0,
+          "fillGradient": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 31
+          },
+          "hiddenSeries": false,
+          "id": 23763572002,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "max": false,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.5.11",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "tidb_domain_lease_expire_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"} - time()",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{instance}}",
+              "metric": "tidb_domain_load_schema_duration_count",
+              "refId": "A",
+              "step": 10
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Lease Duration",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "$$hashKey": "object:541",
+              "format": "dtdurations",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "$$hashKey": "object:542",
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         }
       ],
       "repeat": null,
diff --git a/metrics/metrics.go b/metrics/metrics.go
index bdf5592e2a916..42cdabe20ead7 100644
--- a/metrics/metrics.go
+++ b/metrics/metrics.go
@@ -127,6 +127,7 @@ func RegisterMetrics() {
 	prometheus.MustRegister(KeepAliveCounter)
 	prometheus.MustRegister(LoadPrivilegeCounter)
 	prometheus.MustRegister(InfoCacheCounters)
+	prometheus.MustRegister(LeaseExpireTime)
 	prometheus.MustRegister(LoadSchemaCounter)
 	prometheus.MustRegister(LoadSchemaDuration)
 	prometheus.MustRegister(MetaHistogram)