From 04d068ddf6af2afd2d3e50d534d2c6c75a92d06e Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Tue, 28 Dec 2021 14:37:49 +0800 Subject: [PATCH] This is an automated cherry-pick of #4038 Signed-off-by: ti-chi-bot --- cdc/owner/changefeed.go | 34 +- cdc/owner/metrics.go | 18 +- metrics/grafana/ticdc.json | 6216 +++++++++++++++++++++++++++++------- 3 files changed, 5176 insertions(+), 1092 deletions(-) diff --git a/cdc/owner/changefeed.go b/cdc/owner/changefeed.go index deeadde08b5..cbb5df94266 100644 --- a/cdc/owner/changefeed.go +++ b/cdc/owner/changefeed.go @@ -63,6 +63,8 @@ type changefeed struct { metricsChangefeedCheckpointTsGauge prometheus.Gauge metricsChangefeedCheckpointTsLagGauge prometheus.Gauge + metricsChangefeedResolvedTsGauge prometheus.Gauge + metricsChangefeedResolvedTsLagGauge prometheus.Gauge newDDLPuller func(ctx cdcContext.Context, startTs uint64) (DDLPuller, error) newSink func(ctx cdcContext.Context) (AsyncSink, error) @@ -268,6 +270,18 @@ LOOP: // init metrics c.metricsChangefeedCheckpointTsGauge = changefeedCheckpointTsGauge.WithLabelValues(c.id) c.metricsChangefeedCheckpointTsLagGauge = changefeedCheckpointTsLagGauge.WithLabelValues(c.id) +<<<<<<< HEAD +======= + c.metricsChangefeedResolvedTsGauge = changefeedResolvedTsGauge.WithLabelValues(c.id) + c.metricsChangefeedResolvedTsLagGauge = changefeedResolvedTsLagGauge.WithLabelValues(c.id) + + // create scheduler + c.scheduler, err = c.newScheduler(ctx, checkpointTs) + if err != nil { + return errors.Trace(err) + } + +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) c.initialized = true return nil } @@ -289,10 +303,21 @@ func (c *changefeed) releaseResources() { log.Warn("Closing sink failed in Owner", zap.String("changefeedID", c.state.ID), zap.Error(err)) } c.wg.Wait() +<<<<<<< HEAD +======= + c.scheduler.Close(ctx) + +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) changefeedCheckpointTsGauge.DeleteLabelValues(c.id) changefeedCheckpointTsLagGauge.DeleteLabelValues(c.id) c.metricsChangefeedCheckpointTsGauge = nil c.metricsChangefeedCheckpointTsLagGauge = nil + + changefeedResolvedTsGauge.DeleteLabelValues(c.id) + changefeedResolvedTsLagGauge.DeleteLabelValues(c.id) + c.metricsChangefeedResolvedTsGauge = nil + c.metricsChangefeedResolvedTsLagGauge = nil + c.initialized = false } @@ -471,10 +496,13 @@ func (c *changefeed) updateStatus(currentTs int64, barrierTs model.Ts) { } return status, changed, nil }) - phyTs := oracle.ExtractPhysical(checkpointTs) + phyCkpTs := oracle.ExtractPhysical(checkpointTs) + c.metricsChangefeedCheckpointTsGauge.Set(float64(phyCkpTs)) + c.metricsChangefeedCheckpointTsLagGauge.Set(float64(currentTs-phyCkpTs) / 1e3) - c.metricsChangefeedCheckpointTsGauge.Set(float64(phyTs)) - c.metricsChangefeedCheckpointTsLagGauge.Set(float64(currentTs-phyTs) / 1e3) + phyRTs := oracle.ExtractPhysical(resolvedTs) + c.metricsChangefeedResolvedTsGauge.Set(float64(phyRTs)) + c.metricsChangefeedResolvedTsLagGauge.Set(float64(currentTs-phyRTs) / 1e3) } func (c *changefeed) Close() { diff --git a/cdc/owner/metrics.go b/cdc/owner/metrics.go index 4aeb513649b..bc1bff8d0aa 100644 --- a/cdc/owner/metrics.go +++ b/cdc/owner/metrics.go @@ -28,7 +28,21 @@ var ( Namespace: "ticdc", Subsystem: "owner", Name: "checkpoint_ts_lag", - Help: "checkpoint ts lag of changefeeds", + Help: "checkpoint ts lag of changefeeds in seconds", + }, []string{"changefeed"}) + changefeedResolvedTsGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "owner", + Name: "resolved_ts", + Help: "resolved ts of changefeeds", + }, []string{"changefeed"}) + changefeedResolvedTsLagGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "owner", + Name: "resolved_ts_lag", + Help: "resolved ts lag of changefeeds in seconds", }, []string{"changefeed"}) ownershipCounter = prometheus.NewCounter( prometheus.CounterOpts{ @@ -63,7 +77,9 @@ const ( // InitMetrics registers all metrics used in owner func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(changefeedCheckpointTsGauge) + registry.MustRegister(changefeedResolvedTsGauge) registry.MustRegister(changefeedCheckpointTsLagGauge) + registry.MustRegister(changefeedResolvedTsLagGauge) registry.MustRegister(ownershipCounter) registry.MustRegister(ownerMaintainTableNumGauge) registry.MustRegister(changefeedStatusGauge) diff --git a/metrics/grafana/ticdc.json b/metrics/grafana/ticdc.json index dd30b382e09..b5a3dc5d721 100644 --- a/metrics/grafana/ticdc.json +++ b/metrics/grafana/ticdc.json @@ -125,7 +125,11 @@ "gnetId": null, "graphTooltip": 1, "id": null, +<<<<<<< HEAD "iteration": 1636101093950, +======= + "iteration": 1640447119260, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "links": [], "panels": [ { @@ -196,6 +200,13 @@ "intervalFactor": 1, "legendFormat": "TiKV-{{instance}}", "refId": "B" + }, + { + "expr": "(time() - process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", job=\"pd\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "PD-{{instance}}", + "refId": "C" } ], "thresholds": [], @@ -823,24 +834,22 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The history of PD cluster leadership, leader node has a value that is great than 0", "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 8 }, - "hiddenSeries": false, - "id": 114, + "id": 293, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "max": true, + "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -849,10 +858,6 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -860,35 +865,23 @@ "seriesOverrides": [], "spaceLength": 10, "stack": false, - "steppedLine": false, + "steppedLine": true, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", + "expr": "pd_tso_role{tidb_cluster=\"$tidb_cluster\", dc=\"global\"} > BOOL 0.5", "format": "time_series", + "hide": false, + "interval": "30s", "intervalFactor": 1, - "legendFormat": "p999-{{instance}}", + "legendFormat": "PD-{{instance}}", "refId": "A" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p99-{{instance}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p95-{{instance}}", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Etcd health check duration", + "title": "PD leader history", "tooltip": { "shared": true, "sort": 0, @@ -904,7 +897,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -938,9 +931,10 @@ "x": 0, "y": 1 }, - "id": 266, + "id": 11, "panels": [ { +<<<<<<< HEAD "cards": { "cardPadding": 1, "cardRound": null @@ -954,29 +948,71 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", +======= + "columns": [ + { + "text": "Current", + "value": "current" + } + ], +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "The number of captured table of TiCDC nodes ", + "fontSize": "100%", "gridPos": { +<<<<<<< HEAD "h": 8, "w": 12, +======= + "h": 5, + "w": 7, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "x": 0, - "y": 2 + "y": 3 }, +<<<<<<< HEAD "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, "id": 262, "legend": { "show": true +======= + "id": 4, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], "targets": [ { +<<<<<<< HEAD "exemplar": true, "expr": "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, capture)", "format": "heatmap", @@ -1011,21 +1047,34 @@ "yBucketBound": "auto", "yBucketNumber": null, "yBucketSize": null +======= + "expr": "sum(ticdc_processor_num_of_tables{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{capture}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Changefeed table count", + "transform": "timeseries_aggregations", + "type": "table" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "columns": [ + { + "text": "Current", + "value": "current" + } + ], "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, + "description": "Internal resolved ts of TiCDC nodes", + "fontSize": "100%", "gridPos": { +<<<<<<< HEAD "h": 8, "w": 12, "x": 12, @@ -1041,60 +1090,107 @@ "show": true, "total": false, "values": false +======= + "h": 10, + "w": 7, + "x": 7, + "y": 3 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "id": 90, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true }, - "percentage": false, - "pluginVersion": "6.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "styles": [ + { + "alias": "table", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Metric", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "resolved ts", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "MM-DD HH:mm:ss.SSS", + "decimals": 2, + "pattern": "Current", + "thresholds": [], + "type": "date", + "unit": "short" + } + ], "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "max(ticdc_processor_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed)", + "format": "time_series", "interval": "", - "legendFormat": "{{capture}}-95", - "queryType": "randomWalk", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{changefeed}}", "refId": "A" }, { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, + "expr": "max(ticdc_processor_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed) > 0", + "format": "time_series", "interval": "", - "legendFormat": "{{capture}}-99", + "intervalFactor": 1, + "legendFormat": "checkpoint-{{capture}}-{{changefeed}}", "refId": "B" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "EtcdWorker tick reactor duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "title": "Processor resolved ts", + "transform": "timeseries_aggregations", + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "${DS_TEST-CLUSTER}", + "description": "Internal resolved ts of captured tables", + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 10, + "x": 14, + "y": 3 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "id": 30, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true }, - "yaxes": [ + "styles": [ { +<<<<<<< HEAD "$$hashKey": "object:2612", "format": "s", "label": null, @@ -1111,142 +1207,203 @@ "max": null, "min": null, "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": 1, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", +======= + "alias": "table", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Metric", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "resolved ts", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "MM-DD HH:mm:ss.SSS", + "decimals": 2, + "pattern": "Current", + "thresholds": [], + "type": "date", + "unit": "short" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + } + ], + "targets": [ + { + "expr": "bottomk(10, max(ticdc_processor_table_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed,table))", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{changefeed}}-{{table}}", + "refId": "A" + }, + { + "expr": "bottomk(10, max(ticdc_processor_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed,table) > 0)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "checkpoint-{{capture}}-{{changefeed}}", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Table resolved ts", + "transform": "timeseries_aggregations", + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "The number of replicated tables maintained in owner", + "fontSize": "100%", "gridPos": { +<<<<<<< HEAD "h": 8, "w": 12, "x": 0, "y": 10 +======= + "h": 5, + "w": 7, + "x": 0, + "y": 8 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 256, - "legend": { - "show": true + "id": 138, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, + "expr": "sum(ticdc_owner_maintain_table_num{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\",type=\"total\"}) by (capture)", + "format": "time_series", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "queryType": "randomWalk", + "intervalFactor": 1, + "legendFormat": "{{capture}}-total", "refId": "A" + }, + { + "expr": "sum(ticdc_owner_maintain_table_num{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\",type=\"wip\"}) by (capture)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{capture}}-wip", + "refId": "B" } ], "timeFrom": null, "timeShift": null, - "title": "EtcdWorker exec etcd txn duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "title": "Table count maintained by owner", + "transform": "timeseries_aggregations", + "type": "table" }, { "aliasColors": {}, - "bars": false, + "bars": true, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, + "description": "The checkpoint ts of changefeeds.", + "fill": 0, "gridPos": { +<<<<<<< HEAD "h": 8, "w": 12, "x": 12, "y": 10 +======= + "h": 7, + "w": 9, + "x": 0, + "y": 13 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "hiddenSeries": false, - "id": 258, + "id": 86, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, "total": false, - "values": false + "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, + "links": [], "nullPointMode": "null", - "options": { - "alertThreshold": true - }, "percentage": false, - "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/approximate current time.*/", + "bars": false + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "max(pd_cluster_tso{tidb_cluster=\"$tidb_cluster\"})", "format": "time_series", + "hide": false, "interval": "", +<<<<<<< HEAD "legendFormat": "{{capture}}-p95", "queryType": "randomWalk", "refId": "A" @@ -1257,14 +1414,35 @@ "hide": false, "interval": "", "legendFormat": "{{lcapture}-p99}", +======= + "intervalFactor": 1, + "legendFormat": "approximate current time (s)", + "refId": "A" + }, + { + "expr": "max(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "B" + }, + { + "expr": "max(ticdc_owner_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed) > 0", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-resolvedts", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker exec etcd txn duration percentile", + "title": "Changefeed checkpoint", "tooltip": { "shared": true, "sort": 0, @@ -1273,6 +1451,7 @@ "type": "graph", "xaxis": { "buckets": null, + "max": null, "mode": "time", "name": null, "show": true, @@ -1280,10 +1459,14 @@ }, "yaxes": [ { +<<<<<<< HEAD "$$hashKey": "object:1612", "format": "s", +======= + "format": "dateTimeAsIso", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -1295,7 +1478,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -1304,77 +1487,108 @@ } }, { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolatePurples", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "Request count of etcd operation per second", + "fill": 1, + "fillGradient": 0, "gridPos": { +<<<<<<< HEAD "h": 8, "w": 12, "x": 0, "y": 18 +======= + "h": 7, + "w": 8, + "x": 9, + "y": 13 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 254, + "hiddenSeries": false, + "id": 102, "legend": { - "show": true + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, capture)", - "format": "heatmap", - "instant": false, + "expr": "sum(rate(ticdc_etcd_request_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (capture, type)", + "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{le}}", - "queryType": "randomWalk", + "legendFormat": "{{capture}}-{{type}}", "refId": "A" } ], - "title": "EtcdWorker txn size ", + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "PD etcd requests/s", "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": null, - "type": "heatmap", - "xAxis": { - "show": true + "shared": true, + "sort": 0, + "value_type": "individual" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "decbytes", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -1382,38 +1596,38 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "The number of errors that interrupt changefeed per minute ", "fill": 1, - "fillGradient": 0, "gridPos": { +<<<<<<< HEAD "h": 8, "w": 12, "x": 12, "y": 18 +======= + "h": 7, + "w": 7, + "x": 17, + "y": 13 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "hiddenSeries": false, - "id": 260, + "id": 82, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", - "options": { - "alertThreshold": true - }, "percentage": false, - "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", @@ -1423,27 +1637,18 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "interval": "", - "legendFormat": "{{capture}}-p95", - "queryType": "randomWalk", + "expr": "sum(delta(ticdc_processor_exit_with_error_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, - "interval": "", - "legendFormat": "{{capture}}-p99", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker txn size percentile", + "title": "Exit error count/m", "tooltip": { "shared": true, "sort": 0, @@ -1459,8 +1664,12 @@ }, "yaxes": [ { +<<<<<<< HEAD "$$hashKey": "object:2055", "format": "bytes", +======= + "format": "short", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -1481,6 +1690,7 @@ "align": false, "alignLevel": null } +<<<<<<< HEAD } ], "title": "EtcdWorker", @@ -1606,6 +1816,46 @@ }, "percentage": false, "pluginVersion": "6.1.6", +======= + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "pointradius": 2, "points": false, "renderer": "flot", @@ -1615,6 +1865,7 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", "interval": "", @@ -1628,14 +1879,44 @@ "hide": false, "interval": "", "legendFormat": "{{capture}}-99", +======= + "expr": "max(ticdc_owner_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}", + "refId": "A" + }, + { + "expr": "sum(ticdc_processor_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture,changefeed)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", "refId": "B" + }, + { + "expr": "max(ticdc_owner_resolved_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-resolvedts", + "refId": "C" + }, + { + "expr": "sum(ticdc_processor_resolved_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture,changefeed)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}-resolvedts", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker tick reactor duration", + "title": "Changefeed checkpoint lag", "tooltip": { "shared": true, "sort": 0, @@ -1651,17 +1932,15 @@ }, "yaxes": [ { - "$$hashKey": "object:2612", "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "$$hashKey": "object:2613", - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -1675,122 +1954,45 @@ } }, { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": 1, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "The derivative of a changefeed checkpoint, ideally it should not be 0 (means the changefeed is stopped).", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 256, - "legend": { - "show": true - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "EtcdWorker exec etcd txn duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 10 + "y": 20 }, "hiddenSeries": false, - "id": 258, + "id": 2, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, + "hideEmpty": false, + "hideZero": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { - "alertThreshold": true + "dataLinks": [] }, + "paceLength": 10, "percentage": false, - "pluginVersion": "6.1.6", - "pointradius": 2, + "pointradius": 0.5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -1799,28 +2001,19 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "sum(deriv(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed) / 1000 > 0", "format": "time_series", "interval": "", - "legendFormat": "{{capture}}-p95", - "queryType": "randomWalk", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, - "interval": "", - "legendFormat": "{{capture}-p99}", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker exec etcd txn duration percentile", + "title": "Changefeed checkpoint derivative", "tooltip": { "shared": true, "sort": 0, @@ -1836,16 +2029,14 @@ }, "yaxes": [ { - "$$hashKey": "object:1612", "format": "s", "label": null, "logBase": 2, - "max": null, - "min": null, + "max": "3600", + "min": "0.001", "show": true }, { - "$$hashKey": "object:1613", "format": "short", "label": null, "logBase": 1, @@ -1859,119 +2050,38 @@ "alignLevel": null } }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolatePurples", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 254, - "legend": { - "show": true - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, capture)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "EtcdWorker txn size ", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": null, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "decbytes", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "The status of each changefeed.\n\n0: Normal\n\n1: Error\n\n2: Failed\n\n3: Stopped\n\n4: Finished\n\n-1: Unknown", "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 12, - "y": 18 + "x": 0, + "y": 27 }, - "hiddenSeries": false, - "id": 260, + "id": 163, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, - "min": false, + "current": true, + "max": true, + "min": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", - "options": { - "alertThreshold": true - }, "percentage": false, - "pluginVersion": "6.1.6", - "pointradius": 2, - "points": false, + "pointradius": 1, + "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -1979,27 +2089,19 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "interval": "", - "legendFormat": "{{capture}}-p95", - "queryType": "randomWalk", + "expr": "ticdc_owner_status{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{changefeed}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, - "interval": "", - "legendFormat": "{{capture}}-p99", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker txn size percentile", + "title": "The status of changefeeds", "tooltip": { "shared": true, "sort": 0, @@ -2015,8 +2117,7 @@ }, "yaxes": [ { - "$$hashKey": "object:2055", - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2024,7 +2125,6 @@ "show": true }, { - "$$hashKey": "object:2056", "format": "short", "label": null, "logBase": 1, @@ -2037,36 +2137,1084 @@ "align": false, "alignLevel": null } - } - ], - "title": "EtcdWorker", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 11, - "panels": [ + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Estimate the remaining time for a changefeed catch-up upstream TiDB.", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 27 + }, + "hiddenSeries": false, + "id": 253, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "abs(max(ticdc_owner_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"} / (deriv(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])/1000)) by (changefeed))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Changefeed checkpoint catch-up ETA", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "", + "logBase": 2, + "max": "604800", + "min": "1", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 1, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "Sink write duration of changefeeds", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 34 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 94, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "repeat": null, + "repeatDirection": "h", + "reverseYBuckets": false, + "targets": [ + { + "expr": "max(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Sink write duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Percentiles of sink write duration of changefeeds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 34 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p999", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sink write duration percentile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The number of changed rows that are written to downstream per second", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate(ticdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}", + "refId": "A" + }, + { + "expr": "sum (rate(ticdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, +<<<<<<< HEAD + "title": "EtcdWorker tick reactor duration", +======= + "title": "Sink write rows count/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { +<<<<<<< HEAD + "$$hashKey": "object:2612", + "format": "s", +======= + "format": "none", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2613", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { +<<<<<<< HEAD + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 1, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 256, + "legend": { + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "EtcdWorker exec etcd txn duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "hiddenSeries": false, + "id": 258, + "legend": { + "avg": false, + "current": false, +======= + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Percentiles of sink batch size", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 41 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "6.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { +<<<<<<< HEAD + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "interval": "", + "legendFormat": "{{capture}}-p95", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "hide": false, + "interval": "", + "legendFormat": "{{capture}-p99}", + "refId": "B" +======= + "expr": "histogram_quantile(0.90, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-p90", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-p99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p999", + "refId": "C" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, +<<<<<<< HEAD + "title": "EtcdWorker exec etcd txn duration percentile", +======= + "title": "Sink write batch size percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { +<<<<<<< HEAD + "$$hashKey": "object:1612", + "format": "s", +======= + "format": "none", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1613", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { +<<<<<<< HEAD + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolatePurples", + "exponent": 0.5, +======= + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 +======= + "description": "Asynchronous flush sink duration of changefeeds", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 48 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, +<<<<<<< HEAD + "id": 254, + "legend": { + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, capture)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "EtcdWorker txn size ", +======= + "id": 93, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "expr": "max(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Flush sink duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "show": true, + "showHistogram": true + }, +<<<<<<< HEAD + "tooltipDecimals": null, +======= + "tooltipDecimals": 1, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { +<<<<<<< HEAD + "decimals": null, + "format": "decbytes", +======= + "decimals": 1, + "format": "s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, +<<<<<<< HEAD + "yBucketBound": "auto", +======= + "yBucketBound": "upper", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 260, +======= + "description": "Percentiles of asynchronous flush sink duration of changefeeds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 48 + }, + "hiddenSeries": false, + "id": 98, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { +<<<<<<< HEAD + "alertThreshold": true + }, +======= + "dataLinks": [] + }, + "paceLength": 10, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "percentage": false, + "pluginVersion": "6.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { +<<<<<<< HEAD + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "interval": "", + "legendFormat": "{{capture}}-p95", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "hide": false, + "interval": "", + "legendFormat": "{{capture}}-p99", + "refId": "B" +======= + "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p999", + "refId": "C" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, +<<<<<<< HEAD + "title": "EtcdWorker txn size percentile", +======= + "title": "Flush sink duration percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { +<<<<<<< HEAD + "$$hashKey": "object:2055", + "format": "bytes", +======= + "format": "s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2056", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } +<<<<<<< HEAD + } + ], + "title": "EtcdWorker", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 11, + "panels": [ { "columns": [ { - "text": "Current", - "value": "current" + "text": "Current", + "value": "current" + } + ], + "datasource": "${DS_TEST-CLUSTER}", + "description": "The number of captured table of TiCDC nodes ", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 2 +======= + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 1, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The duration of detecting and waiting conflict of MySQL sink", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 55 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 103, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "repeatDirection": "h", + "reverseYBuckets": false, + "targets": [ + { + "expr": "max(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" } ], + "title": "MySQL sink conflict detect duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of captured table of TiCDC nodes ", - "fontSize": "100%", + "description": "Distribution of MySQL worker loads", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 5, - "w": 7, - "x": 0, - "y": 2 + "h": 7, + "w": 12, + "x": 12, + "y": 55 + }, + "hiddenSeries": false, + "id": 95, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 4, "links": [], @@ -2077,6 +3225,7 @@ "col": 0, "desc": true }, +<<<<<<< HEAD "styles": [ { "alias": "Time", @@ -2102,34 +3251,236 @@ "targets": [ { "expr": "sum(ticdc_processor_num_of_tables{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", +======= + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture,bucket)", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{bucket}}", + "refId": "A" + }, + { + "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) >= 0)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", + "hide": true, "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{capture}}", "refId": "A" +======= + "legendFormat": "total worker", + "refId": "B" + }, + { + "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) <= 2)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "0-2 row/s worker", + "refId": "C" + }, + { + "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) > 2 and rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) <= 10)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "2-10 row/s worker", + "refId": "D" + }, + { + "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) > 10 and rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) <= 100)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "10-100 row/s worker", + "refId": "E" + }, + { + "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) > 100)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ">100 row/s worker", + "refId": "F" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "timeFrom": null, "timeShift": null, +<<<<<<< HEAD "title": "Changefeed table count", "transform": "timeseries_aggregations", "type": "table" }, { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], + "columns": [ +======= + "title": "MySQL sink worker load", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + { + "text": "Current", + "value": "current" + } + ], +<<<<<<< HEAD + "datasource": "${DS_TEST-CLUSTER}", + "description": "Internal resolved ts of TiCDC nodes", + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 7, + "x": 7, + "y": 2 +======= + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 1, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The duration of executing DDL in MySQL sink", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 62 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 165, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "repeatDirection": "h", + "reverseYBuckets": false, + "targets": [ + { + "expr": "max(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "MySQL sink ddl execution duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Internal resolved ts of TiCDC nodes", - "fontSize": "100%", + "description": "Percentiles of detecting and waiting conflict duration of MySQL sink", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 10, - "w": 7, - "x": 7, - "y": 2 + "h": 7, + "w": 12, + "x": 12, + "y": 62 + }, + "hiddenSeries": false, + "id": 83, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 90, "links": [], @@ -2140,6 +3491,7 @@ "col": 0, "desc": true }, +<<<<<<< HEAD "styles": [ { "alias": "table", @@ -2189,22 +3541,83 @@ "intervalFactor": 1, "legendFormat": "checkpoint-{{capture}}-{{changefeed}}", "refId": "B" +======= + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.999,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p999", + "refId": "C" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "timeFrom": null, "timeShift": null, +<<<<<<< HEAD "title": "Processor resolved ts", "transform": "timeseries_aggregations", "type": "table" }, { "columns": [ +======= + "title": "MySQL sink conflict detect duration percentile", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) { "text": "Current", "value": "current" } ], "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Internal resolved ts of captured tables", "fontSize": "100%", "gridPos": { @@ -2212,18 +3625,75 @@ "w": 10, "x": 14, "y": 2 +======= + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 149, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 30, "links": [], +<<<<<<< HEAD "pageSize": null, "scroll": true, "showHeader": true, "sort": { "col": 0, "desc": true +======= + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ capture }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Processor Memory Consumption Per Capture", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "styles": [ { +<<<<<<< HEAD "alias": "table", "colorMode": null, "colors": [ @@ -2238,6 +3708,14 @@ "thresholds": [], "type": "string", "unit": "short" +======= + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "alias": "resolved ts", @@ -2255,13 +3733,68 @@ "unit": "short" } ], +<<<<<<< HEAD "targets": [ { "expr": "bottomk(10, max(ticdc_processor_table_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed,table))", +======= + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Percentiles of executing DDL duration of MySQL sink", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 69 + }, + "hiddenSeries": false, + "id": 166, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{capture}}-{{changefeed}}-{{table}}", "refId": "A" }, @@ -2272,10 +3805,31 @@ "intervalFactor": 1, "legendFormat": "checkpoint-{{capture}}-{{changefeed}}", "refId": "B" +======= + "legendFormat": "{{instance}}-p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.999,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p999", + "refId": "C" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "timeFrom": null, "timeShift": null, +<<<<<<< HEAD "title": "Table resolved ts", "transform": "timeseries_aggregations", "type": "table" @@ -2347,11 +3901,49 @@ "refId": "B" } ], - "timeFrom": null, - "timeShift": null, - "title": "Table count maintained by owner", - "transform": "timeseries_aggregations", - "type": "table" + "timeFrom": null, + "timeShift": null, + "title": "Table count maintained by owner", + "transform": "timeseries_aggregations", + "type": "table" +======= + "title": "MySQL sink ddl execution duration percentile", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "aliasColors": {}, @@ -2360,6 +3952,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The checkpoint ts of changefeeds.", "fill": 0, "gridPos": { @@ -2369,17 +3962,29 @@ "y": 12 }, "id": 86, +======= + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 164, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, "avg": false, - "current": true, + "current": false, "max": false, "min": false, +<<<<<<< HEAD "rightSide": false, +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, "sideWidth": null, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 2, @@ -2400,10 +4005,15 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "max(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed) > 0", +======= + "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture, changefeed)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{changefeed}}", "refId": "A" }, @@ -2415,13 +4025,21 @@ "intervalFactor": 1, "legendFormat": "approximate current time (s)", "refId": "B" +======= + "legendFormat": "{{ capture }}-{{ changefeed }}", + "refId": "A" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Changefeed checkpoint", +======= + "title": "Processor Memory Consumption Per Changefeed", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -2438,7 +4056,11 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "dateTimeAsIso", +======= + "format": "bytes", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -2459,12 +4081,180 @@ "alignLevel": null } }, + { + "columns": [], + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dateTimeAsSystem" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "capture" + }, + "properties": [ + { + "id": "custom.width", + "value": 191 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "changefeed" + }, + "properties": [ + { + "id": "custom.width", + "value": 209 + } + ] + } + ] + }, + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 200, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "last-snap-ts" + } + ] + }, + "pageSize": null, + "pluginVersion": "6.1.6", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Value", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "exemplar": true, + "expr": "ticdc_processor_schema_storage_gc_ts{changefeed=~\"$changefeed\", capture=~\"$capture\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Schema Storage GC progress", + "transform": "table", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "capture", + "changefeed", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value #A": "last-snap-ts" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "last-snap-ts" + } + ] + } + } + ], + "type": "table" + } + ], + "title": "Changefeed", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 206, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Request count of etcd operation per second", "fill": 1, "fillGradient": 0, @@ -2478,13 +4268,28 @@ "id": 102, "legend": { "alignAsTable": true, +======= + "description": "The number of events that puller outputs to sorter \n per second", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "hiddenSeries": false, + "id": 218, + "legend": { + "alignAsTable": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -2507,11 +4312,18 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(rate(ticdc_etcd_request_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (capture, type)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{capture}}-{{type}}", +======= + "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -2519,7 +4331,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "PD etcd requests/s", +======= + "title": "Puller output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -2548,7 +4364,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -2562,22 +4378,36 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of errors that interrupt changefeed per minute ", +======= + "description": "The total number of events that puller outputs", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "gridPos": { +<<<<<<< HEAD "h": 7, "w": 7, "x": 17, "y": 12 }, "id": 82, +======= + "h": 4, + "w": 12, + "x": 12, + "y": 4 + }, + "hiddenSeries": false, + "id": 229, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, - "max": false, + "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -2596,10 +4426,17 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(delta(ticdc_processor_exit_with_error_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{capture}}", +======= + "expr": "sum(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -2607,7 +4444,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Exit error count/m", +======= + "title": "Puller output events", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -2645,6 +4486,7 @@ } }, { +<<<<<<< HEAD "aliasColors": {}, "bars": false, "dashLength": 10, @@ -2748,31 +4590,49 @@ } }, { +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The derivative of a changefeed checkpoint, ideally it should not be 0 (means the changefeed is stopped).", +======= + "description": "The number of events that sorter outputs to puller \n per second", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 4, "w": 12, +<<<<<<< HEAD "x": 12, "y": 19 }, "hiddenSeries": false, "id": 2, +======= + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 228, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, +<<<<<<< HEAD "hideEmpty": false, "hideZero": false, "max": false, +======= + "max": true, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -2795,11 +4655,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(deriv(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed) / 1000 > 0", +======= + "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture, type)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{changefeed}}", +======= + "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -2807,7 +4675,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Changefeed checkpoint derivative", +======= + "title": "Sorter output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -2823,6 +4695,7 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "s", "label": null, "logBase": 2, @@ -2832,11 +4705,15 @@ }, { "format": "short", +======= + "format": "none", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, "min": null, "show": true +<<<<<<< HEAD } ], "yaxis": { @@ -2917,6 +4794,8 @@ "max": null, "min": null, "show": true +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "format": "short", @@ -2924,7 +4803,11 @@ "logBase": 1, "max": null, "min": null, +<<<<<<< HEAD "show": true +======= + "show": false +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "yaxis": { @@ -2938,6 +4821,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Estimate the remaining time for a changefeed catch-up upstream TiDB.", "fieldConfig": { "defaults": { @@ -2945,18 +4829,30 @@ }, "overrides": [] }, +======= + "description": "The total number of events that sorter outputs", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 12, +<<<<<<< HEAD "y": 26 }, "hiddenSeries": false, "id": 253, "legend": { - "alignAsTable": true, + "alignAsTable": true, +======= + "y": 8 + }, + "hiddenSeries": false, + "id": 220, + "legend": { + "alignAsTable": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "avg": false, "current": true, "hideEmpty": true, @@ -2986,12 +4882,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "exemplar": true, "expr": "abs(max(ticdc_owner_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"} / (deriv(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])/1000)) by (changefeed))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{changefeed}}", +======= + "expr": "sum(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -2999,7 +4902,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Changefeed checkpoint catch-up ETA", +======= + "title": "Sorter output events", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -3015,11 +4922,19 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "s", "label": "", "logBase": 2, "max": "604800", "min": "1", +======= + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true }, { @@ -3037,46 +4952,50 @@ } }, { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": 1, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Sink write duration of changefeeds", +======= + "description": "The number of events that mounter outputs to sink per second", + "fill": 1, + "fillGradient": 0, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 0, +<<<<<<< HEAD "y": 33 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, "id": 94, +======= + "y": 12 + }, + "hiddenSeries": false, + "id": 219, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, + "lines": true, + "linewidth": 1, "links": [], +<<<<<<< HEAD "repeat": null, "repeatDirection": "h", "reverseYBuckets": false, @@ -3091,29 +5010,70 @@ } ], "title": "Sink write duration", - "tooltip": { - "show": true, - "showHistogram": true +======= + "nullPointMode": "null", + "options": { + "dataLinks": [] }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mounter output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -3121,19 +5081,30 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Percentiles of sink write duration of changefeeds", +======= + "description": "The total number of events that mounter outputs", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 12, +<<<<<<< HEAD "y": 33 }, "hiddenSeries": false, "id": 35, +======= + "y": 12 + }, + "hiddenSeries": false, + "id": 224, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, "max": false, @@ -3161,11 +5132,16 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", +======= + "expr": "sum(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{instance}}-p95", + "legendFormat": "{{changefeed}}-{{capture}}", "refId": "A" +<<<<<<< HEAD }, { "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", @@ -3180,13 +5156,19 @@ "intervalFactor": 1, "legendFormat": "{{instance}}-p999", "refId": "C" +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Sink write duration percentile", +======= + "title": "Mounter output events", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -3202,9 +5184,9 @@ }, "yaxes": [ { - "format": "s", + "format": "none", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -3229,6 +5211,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of changed rows that are written to downstream per second", "fill": 1, "fillGradient": 0, @@ -3350,6 +5333,26 @@ "max": false, "min": false, "rightSide": true, +======= + "description": "The number of events that table sorter outputs to buffer sink per second", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 223, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, "total": false, "values": true @@ -3372,10 +5375,17 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "histogram_quantile(0.90, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{capture}}-p90", +======= + "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { @@ -3398,7 +5408,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Sink write batch size percentile", +======= + "title": "Table sink output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -3427,7 +5441,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -3518,19 +5532,30 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Percentiles of asynchronous flush sink duration of changefeeds", +======= + "description": "The total number of events that table sinks emit", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 12, +<<<<<<< HEAD "y": 47 }, "hiddenSeries": false, "id": 98, +======= + "y": 16 + }, + "hiddenSeries": false, + "id": 221, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, "max": false, @@ -3558,6 +5583,7 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", "format": "time_series", "intervalFactor": 1, @@ -3577,13 +5603,24 @@ "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}-p999", "refId": "C" +======= + "expr": "sum(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", + "refId": "A" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Flush sink duration percentile", +======= + "title": "Table sink output events", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -3599,9 +5636,9 @@ }, "yaxes": [ { - "format": "s", + "format": "none", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -3704,6 +5741,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Distribution of MySQL worker loads", "fill": 1, "fillGradient": 0, @@ -3718,6 +5756,22 @@ "legend": { "alignAsTable": true, "avg": true, +======= + "description": "The total number of events that buffer sink outputs to backend sink per second", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 222, + "legend": { + "alignAsTable": false, + "avg": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "current": true, "max": true, "min": false, @@ -3735,6 +5789,10 @@ "options": { "dataLinks": [] }, +<<<<<<< HEAD +======= + "paceLength": 10, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "percentage": false, "pointradius": 2, "points": false, @@ -3745,12 +5803,20 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture,bucket)", +======= + "expr": "sum(rate(ticdc_sink_buffer_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "hide": true, "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{capture}}-{{bucket}}", +======= + "legendFormat": "{{changefeed}}-{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { @@ -3802,7 +5868,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "MySQL sink worker load", +======= + "title": "Buffer sink output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -3818,7 +5888,11 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "short", +======= + "format": "none", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -3831,7 +5905,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -3840,6 +5914,7 @@ } }, { +<<<<<<< HEAD "cards": { "cardPadding": 0, "cardRound": 0 @@ -3868,10 +5943,31 @@ "id": 165, "legend": { "alignAsTable": true, +======= + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The total number of events that buffer sink emits", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 226, + "legend": { + "alignAsTable": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "avg": false, "current": true, "max": true, "min": false, +<<<<<<< HEAD "rightSide": true, "show": true, "sort": "current", @@ -3911,11 +6007,83 @@ "max": null, "min": null, "show": true, - "splitFactor": null + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null +======= + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(ticdc_sink_buffer_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Buffer sink output events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "aliasColors": {}, @@ -3923,17 +6091,29 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Percentiles of detecting and waiting conflict duration of MySQL sink", +======= + "description": "The number of rows(events) that are processed by sink per second", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, +<<<<<<< HEAD "x": 12, "y": 61 }, "hiddenSeries": false, "id": 83, +======= + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 227, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, @@ -3942,6 +6122,8 @@ "min": false, "rightSide": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -3962,11 +6144,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-p95", +======= + "expr": "sum (rate(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { @@ -3990,7 +6180,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "MySQL sink conflict detect duration percentile", +======= + "title": "Sink output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -4033,20 +6227,34 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD +======= + "description": "The number of rows(events) that are processed by sink", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "gridPos": { "h": 8, "w": 12, +<<<<<<< HEAD "x": 0, "y": 68 }, "id": 149, +======= + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 225, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": false }, @@ -4054,6 +6262,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", +<<<<<<< HEAD +======= + "options": { + "dataLinks": [] + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "percentage": false, "pointradius": 2, "points": false, @@ -4064,10 +6278,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture)", +======= + "expr": "sum(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", + "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{ capture }}", +======= + "legendFormat": "{{changefeed}}-{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -4075,10 +6298,14 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Processor Memory Consumption Per Capture", +======= + "title": "Sink output events", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -4118,17 +6345,29 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Percentiles of executing DDL duration of MySQL sink", +======= + "description": "The number of rows that sink flushes to downstream per second", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, +<<<<<<< HEAD "x": 12, "y": 68 }, "hiddenSeries": false, "id": 166, +======= + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 108, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, @@ -4137,6 +6376,8 @@ "min": false, "rightSide": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -4157,11 +6398,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", +======= + "expr": "sum(rate(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{instance}}-p95", +======= + "legendFormat": "{{changefeed}}-{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { @@ -4185,7 +6434,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "MySQL sink ddl execution duration percentile", +======= + "title": "Sink flush rows/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -4209,7 +6462,7 @@ "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -4228,20 +6481,33 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD +======= + "description": "The number of rows(events) that are flushed by sink", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, +<<<<<<< HEAD "y": 75 }, "id": 164, +======= + "y": 28 + }, + "hiddenSeries": false, + "id": 240, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": false }, @@ -4249,6 +6515,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", +<<<<<<< HEAD +======= + "options": { + "dataLinks": [] + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "percentage": false, "pointradius": 2, "points": false, @@ -4259,10 +6531,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture, changefeed)", +======= + "expr": "sum(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed, capture)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", + "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{ capture }}-{{ changefeed }}", +======= + "legendFormat": "{{changefeed}}-{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -4270,10 +6551,14 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Processor Memory Consumption Per Changefeed", +======= + "title": "Sink flush rows", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -4306,7 +6591,21 @@ "align": false, "alignLevel": null } - }, + } + ], + "title": "Dataflow", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 13, + "panels": [ { "columns": [], "datasource": "${DS_TEST-CLUSTER}", @@ -4480,24 +6779,184 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of events that puller outputs to sorter \n per second", +======= + "description": "The number of established Eventfeed RPC between TiCDC and TiKV", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 4, - "w": 12, + "h": 7, + "w": 8, "x": 0, +<<<<<<< HEAD "y": 3 }, "hiddenSeries": false, "id": 218, +======= + "y": 5 + }, + "hiddenSeries": false, + "id": 15, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, - "max": true, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*-rpc/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { +<<<<<<< HEAD + "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}", +======= + "expr": "sum(ticdc_kvclient_event_feed_count{tidb_cluster=\"$tidb_cluster\"}) by (instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "refId": "A" + }, + { + "expr": "sum(grpc_client_started_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance) - sum(grpc_client_handled_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-rpc", + "refId": "B" + }, + { + "expr": "sum(grpc_client_started_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}-rpc-started", + "refId": "C" + }, + { + "expr": "sum(grpc_client_handled_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}-rpc-handled", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, +<<<<<<< HEAD + "title": "Puller output events/s", +======= + "title": "Eventfeed count", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD + "description": "The total number of events that puller outputs", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 3 + }, + "hiddenSeries": false, + "id": 229, +======= + "decimals": null, + "description": "Percentiles of Eventfeed message size", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 5 + }, + "hiddenSeries": false, + "id": 17, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -4520,18 +6979,36 @@ "steppedLine": false, "targets": [ { - "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", +<<<<<<< HEAD + "expr": "sum(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance, type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", +======= + "expr": "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p999", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p95", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Puller output events/s", +<<<<<<< HEAD + "title": "Puller output events", +======= + "title": "Event size percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4547,7 +7024,7 @@ }, "yaxes": [ { - "format": "none", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -4560,7 +7037,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -4570,28 +7047,44 @@ }, { "aliasColors": {}, - "bars": false, + "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The total number of events that puller outputs", +<<<<<<< HEAD + "description": "The number of events that sorter outputs to puller \n per second", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 12, - "y": 3 + "x": 0, + "y": 7 }, "hiddenSeries": false, - "id": 229, + "id": 228, +======= + "description": "The number of errors that interrupt Eventfeed RPC", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 5 + }, + "hiddenSeries": false, + "id": 28, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -4614,18 +7107,50 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance, type)", +<<<<<<< HEAD + "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed,type)", +======= + "expr": "sum(increase(ticdc_kvclient_event_feed_error_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (type)", "format": "time_series", + "hide": false, + "interval": "1m", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "{{type}}", + "refId": "A" + }, + { + "expr": "-sum(increase(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))", + "format": "time_series", + "hide": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "transfer-leader", + "refId": "B" + }, + { + "expr": "-sum(increase(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, +<<<<<<< HEAD + "legendFormat": "{{capture}}-{{type}}", "refId": "A" +======= + "legendFormat": "move-region", + "refId": "C" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Puller output events", +<<<<<<< HEAD + "title": "Sorter output events/s", +======= + "title": "Eventfeed error/m", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4641,7 +7166,7 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -4668,21 +7193,37 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of events that sorter outputs to puller \n per second", +<<<<<<< HEAD + "description": "The total number of events that sorter outputs", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 0, + "x": 12, "y": 7 }, "hiddenSeries": false, - "id": 228, + "id": 220, +======= + "description": "The number of KV client received events from TiKV per seconds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 29, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, "rightSide": false, @@ -4708,10 +7249,17 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed,type)", +<<<<<<< HEAD + "expr": "sum(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (capture, changefeed, type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{capture}}-{{type}}", +======= + "expr": "sum(rate(ticdc_kvclient_pull_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -4719,7 +7267,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Sorter output events/s", +<<<<<<< HEAD + "title": "Sorter output events", +======= + "title": "KV client receive events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4735,7 +7287,7 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -4748,7 +7300,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -4762,19 +7314,33 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The total number of events that sorter outputs", +<<<<<<< HEAD + "description": "The number of events that mounter outputs to sink per second", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 12, - "y": 7 + "x": 0, + "y": 11 }, "hiddenSeries": false, - "id": 220, + "id": 219, +======= + "description": "The number of events that puller outputs to sorter \n per second", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 5, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, "max": true, @@ -4802,10 +7368,17 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (capture, changefeed, type)", +<<<<<<< HEAD + "expr": "sum(rate(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{type}}", + "legendFormat": "{{capture}}", +======= + "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -4813,7 +7386,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Sorter output events", +<<<<<<< HEAD + "title": "Mounter output events/s", +======= + "title": "Puller output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4856,21 +7433,36 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of events that mounter outputs to sink per second", +<<<<<<< HEAD + "description": "The total number of events that mounter outputs", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 0, + "x": 12, "y": 11 }, "hiddenSeries": false, - "id": 219, + "id": 224, +======= + "description": "The number of events that are buffered in Processor's output channel and Mounter input channel", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 107, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, "rightSide": false, @@ -4896,18 +7488,33 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", +<<<<<<< HEAD + "expr": "sum(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance)", +======= + "expr": "sum(ticdc_mounter_input_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}", + "legendFormat": "{{capture}}-mounter input chan", "refId": "A" + }, + { + "expr": "-sum(ticdc_sink_buffer_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-sink buffer chan", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Mounter output events/s", +<<<<<<< HEAD + "title": "Mounter output events", +======= + "title": "Sink/Mounter buffer size", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4923,7 +7530,7 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -4936,7 +7543,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -4944,30 +7551,119 @@ "alignLevel": null } }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "Duration of sorting unsorted events", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 19 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Entry sorter sort duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The total number of events that mounter outputs", +<<<<<<< HEAD + "description": "The number of events that table sorter outputs to buffer sink per second", +======= + "description": "Percentiles of sorting events duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 4, + "h": 7, "w": 12, +<<<<<<< HEAD + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 223, +======= "x": 12, - "y": 11 + "y": 19 }, "hiddenSeries": false, - "id": 224, + "id": 53, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -4990,18 +7686,37 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance)", +<<<<<<< HEAD + "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", +======= + "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-p999", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p95", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Mounter output events", +<<<<<<< HEAD + "title": "Table sink output events/s", +======= + "title": "Entry sorter sort duration percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -5017,9 +7732,9 @@ }, "yaxes": [ { - "format": "none", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -5039,98 +7754,80 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of events that table sorter outputs to buffer sink per second", - "fill": 1, - "fillGradient": 0, + "description": "Duration of merging sorted events", "gridPos": { - "h": 4, + "h": 7, "w": 12, "x": 0, - "y": 15 + "y": 26 }, - "hiddenSeries": false, - "id": 223, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 105, "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", + "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Table sink output events/s", + "title": "Entry sorter merge duration", "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "show": true, + "showHistogram": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null }, { "aliasColors": {}, @@ -5138,24 +7835,36 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The total number of events that table sinks emit", +======= + "description": "Percentiles of merging sorted events duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 4, + "h": 7, "w": 12, "x": 12, +<<<<<<< HEAD "y": 15 }, "hiddenSeries": false, "id": 221, +======= + "y": 26 + }, + "hiddenSeries": false, + "id": 106, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -5178,18 +7887,37 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", +======= + "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-p999", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p95", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Table sink output events", +======= + "title": "Entry sorter merge duration percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -5205,9 +7933,9 @@ }, "yaxes": [ { - "format": "none", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -5227,36 +7955,57 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The total number of events that buffer sink outputs to backend sink per second", "fill": 1, "fillGradient": 0, +======= + "description": "Duration of unmarshal events from kv to SQL row", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "gridPos": { - "h": 4, + "h": 7, "w": 12, "x": 0, +<<<<<<< HEAD "y": 19 }, "hiddenSeries": false, "id": 222, +======= + "y": 33 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 101, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true }, - "lines": true, - "linewidth": 1, "links": [], +<<<<<<< HEAD "nullPointMode": "null", "options": { "dataLinks": [] @@ -5288,15 +8037,41 @@ "shared": true, "sort": 0, "value_type": "individual" +======= + "reverseYBuckets": false, + "targets": [ + { + "expr": "max(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Mounter unmarshal duration", + "tooltip": { + "show": true, + "showHistogram": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, +<<<<<<< HEAD "yaxes": [ { "format": "none", @@ -5319,6 +8094,11 @@ "align": false, "alignLevel": null } +======= + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "aliasColors": {}, @@ -5326,24 +8106,36 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The total number of events that buffer sink emits", "fill": 1, +======= + "description": "Percentiles of unmarshal events from kv to SQL row duration", + "fill": 0, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fillGradient": 0, "gridPos": { - "h": 4, + "h": 7, "w": 12, "x": 12, +<<<<<<< HEAD "y": 19 }, "hiddenSeries": false, "id": 226, +======= + "y": 33 + }, + "hiddenSeries": false, + "id": 55, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -5366,18 +8158,39 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_sink_buffer_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", +======= + "expr": "histogram_quantile(0.99, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p99", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" + }, + { + "expr": "histogram_quantile(0.999, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", + "format": "time_series", + "hide": true, + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p999", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Buffer sink output events", +======= + "title": "Mounter unmarshal duration percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -5393,7 +8206,7 @@ }, "yaxes": [ { - "format": "none", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -5401,7 +8214,11 @@ "show": true }, { +<<<<<<< HEAD "format": "short", +======= + "format": "none", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -5420,6 +8237,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of rows(events) that are processed by sink per second", "fill": 1, "fillGradient": 0, @@ -5431,10 +8249,28 @@ }, "hiddenSeries": false, "id": 227, +======= + "description": "The number of KV client dispatched event per second", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 31, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": false, "avg": false, "current": true, +<<<<<<< HEAD +======= + "hideEmpty": true, + "hideZero": true, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": true, "min": false, "rightSide": false, @@ -5455,25 +8291,54 @@ "pointradius": 2, "points": false, "renderer": "flot", +<<<<<<< HEAD "seriesOverrides": [], +======= + "seriesOverrides": [ + { + "alias": "/.*batch-resolved/", + "yaxis": 2 + } + ], +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum (rate(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture)", +======= + "expr": "sum(rate(ticdc_kvclient_send_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture, changefeed, type)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{capture}}", "refId": "A" +======= + "legendFormat": "{{capture}}-{{changefeed}}-{{type}}", + "refId": "A" + }, + { + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture, changefeed, table)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{changefeed}}-batch-resolved", + "refId": "B" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Sink output events/s", +======= + "title": "KV client dispatch events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -5510,12 +8375,89 @@ "alignLevel": null } }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The size of batch resolved ts message from TiKV", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 40 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 97, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "KV client batch resolved size", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "none", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of rows(events) that are processed by sink", "fill": 1, "fillGradient": 0, @@ -5527,10 +8469,28 @@ }, "hiddenSeries": false, "id": 225, +======= + "description": "The number of regions that are being scanned", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 177, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": false, "avg": false, "current": true, +<<<<<<< HEAD +======= + "hideEmpty": true, + "hideZero": true, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": true, "min": false, "rightSide": false, @@ -5557,11 +8517,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (capture, type)", +======= + "expr": "sum(ticdc_kvclient_region_token{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{capture}}", +======= + "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -5569,7 +8537,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Sink output events", +======= + "title": "KV client scanning regions", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -5585,7 +8557,11 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "none", +======= + "format": "short", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -5612,6 +8588,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of rows that sink flushes to downstream per second", "fill": 1, "fillGradient": 0, @@ -5623,6 +8600,19 @@ }, "hiddenSeries": false, "id": 108, +======= + "description": "Active stream count of each gRPC connection", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 188, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": false, "avg": false, @@ -5653,10 +8643,17 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(rate(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{capture}}", +======= + "expr": "sum(ticdc_kvclient_grpc_stream_count{tidb_cluster=\"$tidb_cluster\"}) by (store)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{store}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -5664,7 +8661,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Sink flush rows/s", +======= + "title": "KV client gRPC stream count", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -5707,6 +8708,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of rows(events) that are flushed by sink", "fill": 1, "fillGradient": 0, @@ -5718,13 +8720,32 @@ }, "hiddenSeries": false, "id": 240, +======= + "description": "The number of regions that have not connected to TiKV", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 251, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": false, "avg": false, "current": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sort": "current", "sortDesc": true, @@ -5736,7 +8757,7 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, "pointradius": 2, @@ -5748,11 +8769,20 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (capture, type)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{capture}}", +======= + "exemplar": true, + "expr": "sum(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -5760,7 +8790,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Sink flush rows", +======= + "title": "KV client cached regions", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -5817,23 +8851,48 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of established Eventfeed RPC between TiCDC and TiKV", +======= + "description": "Estimate the remaining time for a changefeed initialization (on a specific capture)", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 8, "x": 0, "y": 32 }, "hiddenSeries": false, "id": 15, +======= + "w": 12, + "x": 12, + "y": 54 + }, + "hiddenSeries": false, + "id": 252, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, "current": true, +<<<<<<< HEAD "hideEmpty": false, "max": false, +======= + "hideEmpty": true, + "hideZero": true, + "max": true, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "min": false, "rightSide": true, "show": true, @@ -5845,7 +8904,7 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "paceLength": 10, "percentage": false, @@ -5863,10 +8922,20 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_kvclient_event_feed_count{tidb_cluster=\"$tidb_cluster\"}) by (instance)", +======= + "exemplar": true, + "expr": "abs(sum(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"} / deriv(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture, changefeed, store))", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", + "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{instance}}", +======= + "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { @@ -5897,7 +8966,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Eventfeed count", +======= + "title": "Estimate remaining time for initialization", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -5913,9 +8986,15 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "short", "label": null, "logBase": 1, +======= + "format": "s", + "label": "", + "logBase": 2, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": null, "min": null, "show": true @@ -5933,18 +9012,35 @@ "align": false, "alignLevel": null } - }, + } + ], + "title": "Events", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 130, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "decimals": null, "description": "Percentiles of Eventfeed message size", +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, - "fillGradient": 0, "gridPos": { +<<<<<<< HEAD "h": 7, "w": 8, "x": 8, @@ -5952,25 +9048,34 @@ }, "hiddenSeries": false, "id": 17, +======= + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 131, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, "avg": false, +<<<<<<< HEAD "current": true, "max": false, "min": false, "rightSide": true, +======= + "current": false, + "max": false, + "min": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -5981,6 +9086,7 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", "format": "time_series", "intervalFactor": 1, @@ -5993,13 +9099,24 @@ "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}-p95", "refId": "B" +======= + "expr": "sum(rate(ticdc_sorter_consume_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{changefeed}}", + "refId": "A" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Event size percentile", +======= + "title": "Unified Sorter intake rate", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -6037,6 +9154,7 @@ } }, { +<<<<<<< HEAD "aliasColors": {}, "bars": true, "dashLength": 10, @@ -6152,15 +9270,20 @@ } }, { +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of KV client received events from TiKV per seconds", +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, - "fillGradient": 0, "gridPos": { +<<<<<<< HEAD "h": 7, "w": 8, "x": 0, @@ -6168,27 +9291,36 @@ }, "hiddenSeries": false, "id": 29, +======= + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 132, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, "avg": false, +<<<<<<< HEAD "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": false, +======= + "current": false, + "max": false, + "min": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -6199,10 +9331,17 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(rate(ticdc_kvclient_pull_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", +======= + "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{changefeed}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -6210,7 +9349,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "KV client receive events/s", +======= + "title": "Unified Sorter event output rate", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -6253,10 +9396,13 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of events that puller outputs to sorter \n per second", +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, - "fillGradient": 0, "gridPos": { +<<<<<<< HEAD "h": 7, "w": 8, "x": 8, @@ -6271,18 +9417,27 @@ "max": true, "min": false, "rightSide": false, +======= + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 133, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -6293,10 +9448,17 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", +======= + "expr": "sum(ticdc_sorter_on_disk_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -6304,7 +9466,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Puller output events/s", +======= + "title": "Unified Sorter on disk data size", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -6320,12 +9486,146 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "none", +======= + "format": "bytes", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD + "description": "The number of events that are buffered in Processor's output channel and Mounter input channel", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "hiddenSeries": false, + "id": 107, +======= + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 134, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legend": { + "avg": false, +<<<<<<< HEAD + "current": true, + "hideEmpty": false, + "max": true, + "min": false, + "rightSide": false, +======= + "current": false, + "max": false, + "min": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { +<<<<<<< HEAD + "expr": "sum(ticdc_mounter_input_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-mounter input chan", + "refId": "A" + }, + { + "expr": "-sum(ticdc_sink_buffer_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-sink buffer chan", + "refId": "B" +======= + "expr": "sum(ticdc_sorter_in_memory_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}", + "refId": "A" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, +<<<<<<< HEAD + "title": "Sink/Mounter buffer size", +======= + "title": "Unified Sorter in-memory data size", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { +<<<<<<< HEAD + "format": "short", +======= + "format": "bytes", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true }, { "format": "short", @@ -6341,23 +9641,117 @@ "alignLevel": null } }, + { + "cards": { +<<<<<<< HEAD + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "Duration of sorting unsorted events", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 46 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Entry sorter sort duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of events that are buffered in Processor's output channel and Mounter input channel", - "fill": 1, - "fillGradient": 0, + "description": "Percentiles of sorting events duration", + "fill": 1, + "fillGradient": 0, +======= + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 39 + "h": 8, + "w": 12, +<<<<<<< HEAD + "x": 12, + "y": 46 }, "hiddenSeries": false, - "id": 107, + "id": 53, "legend": { "alignAsTable": true, "avg": false, @@ -6365,14 +9759,24 @@ "hideEmpty": false, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true +======= + "x": 0, + "y": 21 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 135, + "legend": { + "show": false +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "lines": true, - "linewidth": 1, "links": [], +<<<<<<< HEAD "nullPointMode": "null", "options": { "dataLinks": [] @@ -6388,43 +9792,63 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_mounter_input_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-mounter input chan", + "legendFormat": "{{capture}}-p999", "refId": "A" }, { - "expr": "-sum(ticdc_sink_buffer_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "{{capture}}-sink buffer chan", + "legendFormat": "{{capture}}-p95", "refId": "B" +======= + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(ticdc_sorter_flush_count_histogram_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "A" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Sink/Mounter buffer size", +<<<<<<< HEAD + "title": "Entry sorter sort duration percentile", +======= + "title": "Unified Sorter flush sizes", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "show": true, + "showHistogram": false }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "none", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, +<<<<<<< HEAD "yaxes": [ { - "format": "short", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -6442,64 +9866,78 @@ "align": false, "alignLevel": null } +======= + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "cards": { - "cardPadding": 0, - "cardRound": 0 + "cardPadding": null, + "cardRound": null }, "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateBlues", "exponent": 0.5, - "min": 0, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "Duration of sorting unsorted events", +<<<<<<< HEAD + "description": "Duration of merging sorted events", +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "gridPos": { - "h": 7, + "h": 8, "w": 12, +<<<<<<< HEAD "x": 0, - "y": 46 + "y": 53 +======= + "x": 12, + "y": 21 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 99, +<<<<<<< HEAD + "id": 105, +======= + "id": 136, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true + "show": false }, "links": [], "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", +<<<<<<< HEAD + "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", +======= + "expr": "sum(rate(ticdc_sorter_merge_count_histogram_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (le)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "heatmap", - "instant": false, - "intervalFactor": 2, + "intervalFactor": 1, "legendFormat": "{{le}}", "refId": "A" } ], - "title": "Entry sorter sort duration", +<<<<<<< HEAD + "title": "Entry sorter merge duration", +======= + "timeFrom": null, + "timeShift": null, + "title": "Unified Sorter merge size", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "show": true, - "showHistogram": true + "showHistogram": false }, - "tooltipDecimals": 1, "type": "heatmap", "xAxis": { "show": true @@ -6507,103 +9945,290 @@ "xBucketNumber": null, "xBucketSize": null, "yAxis": { +<<<<<<< HEAD "decimals": 1, "format": "s", +======= + "decimals": null, + "format": "none", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "logBase": 1, "max": null, "min": null, "show": true, "splitFactor": null }, - "yBucketBound": "upper", + "yBucketBound": "auto", "yBucketNumber": null, "yBucketSize": null + } + ], + "title": "Unified Sorter", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 266, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD + "description": "Percentiles of merging sorted events duration", +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, +<<<<<<< HEAD + "x": 12, + "y": 53 + }, + "hiddenSeries": false, + "id": 106, +======= + "x": 0, + "y": 16 + }, + "id": 289, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { +<<<<<<< HEAD + "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", +======= + "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{tidb_cluster=\"$tidb_cluster\", job=\"pd\"}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "format": "time_series", + "hide": false, + "intervalFactor": 1, +<<<<<<< HEAD + "legendFormat": "{{capture}}-p999", +======= + "legendFormat": "{{instance}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p95", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, +<<<<<<< HEAD + "title": "Entry sorter merge duration percentile", +======= + "title": "Etcd MVCC DB total size", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { +<<<<<<< HEAD + "format": "s", +======= + "format": "decbytes", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "Percentiles of sorting events duration", +<<<<<<< HEAD + "description": "Duration of unmarshal events from kv to SQL row", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 60 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 101, +======= "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 46 + "y": 16 }, "hiddenSeries": false, - "id": 53, + "id": 114, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, +<<<<<<< HEAD +======= + "hideEmpty": true, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": true, "min": false, "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "reverseYBuckets": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", +<<<<<<< HEAD + "expr": "max(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", +======= + "expr": "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-p999", + "legendFormat": "p999-{{instance}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "{{capture}}-p95", + "legendFormat": "p99-{{instance}}", "refId": "B" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p95-{{instance}}", + "refId": "C" } ], +<<<<<<< HEAD + "title": "Mounter unmarshal duration", +======= "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Entry sorter sort duration percentile", + "title": "Etcd health check duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "show": true, + "showHistogram": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, +<<<<<<< HEAD + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null +======= "yaxes": [ { "format": "s", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -6621,6 +10246,7 @@ "align": false, "alignLevel": null } +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "cards": { @@ -6628,26 +10254,28 @@ "cardRound": 0 }, "color": { - "cardColor": "#FF9830", - "colorScale": "linear", + "cardColor": "#b4ff00", + "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, - "min": 0, + "max": null, + "min": 1, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "Duration of merging sorted events", +<<<<<<< HEAD + "description": "Percentiles of unmarshal events from kv to SQL row duration", + "fill": 0, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 53 + "x": 12, + "y": 60 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 105, + "hiddenSeries": false, + "id": 55, "legend": { "alignAsTable": true, "avg": false, @@ -6661,22 +10289,78 @@ "total": false, "values": true }, + "lines": true, + "linewidth": 1, "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] +======= + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 267, + "legend": { + "show": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + }, + "pluginVersion": "6.1.6", "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", +<<<<<<< HEAD + "expr": "histogram_quantile(0.99, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p99", +======= + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", "format": "heatmap", "instant": false, + "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", + "queryType": "randomWalk", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" + }, + { + "expr": "histogram_quantile(0.999, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", + "format": "time_series", + "hide": true, + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p999", + "refId": "B" } ], - "title": "Entry sorter merge duration", + "timeFrom": null, + "timeShift": null, +<<<<<<< HEAD + "title": "Mounter unmarshal duration percentile", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" +======= + "title": "EtcdWorker exec etcd txn duration", "tooltip": { "show": true, "showHistogram": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "tooltipDecimals": 1, "type": "heatmap", @@ -6694,9 +10378,34 @@ "show": true, "splitFactor": null }, +<<<<<<< HEAD + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } +======= "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "aliasColors": {}, @@ -6704,59 +10413,91 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Percentiles of merging sorted events duration", +<<<<<<< HEAD + "description": "The number of KV client dispatched event per second", +======= + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, +<<<<<<< HEAD + "x": 0, + "y": 67 + }, + "hiddenSeries": false, + "id": 31, +======= "x": 12, - "y": 53 + "y": 23 }, "hiddenSeries": false, - "id": 106, + "id": 264, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": false, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "dataLinks": [] }, - "paceLength": 10, "percentage": false, + "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*batch-resolved/", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", +<<<<<<< HEAD + "expr": "sum(rate(ticdc_kvclient_send_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture, changefeed, type)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-p999", + "legendFormat": "{{capture}}-{{changefeed}}-{{type}}", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture, changefeed, table)", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "{{capture}}-p95", + "legendFormat": "{{capture}}-{{changefeed}}-batch-resolved", +======= + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "interval": "", + "legendFormat": "{{capture}}-95", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "hide": false, + "interval": "", + "legendFormat": "{{capture}}-99", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "B" } ], @@ -6764,7 +10505,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Entry sorter merge duration percentile", +<<<<<<< HEAD + "title": "KV client dispatch events/s", +======= + "title": "EtcdWorker tick reactor duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -6780,15 +10525,19 @@ }, "yaxes": [ { +<<<<<<< HEAD + "format": "short", +======= "format": "s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -6801,58 +10550,60 @@ "alignLevel": null } }, +<<<<<<< HEAD +======= { "cards": { "cardPadding": 0, "cardRound": 0 }, "color": { - "cardColor": "#FF9830", - "colorScale": "linear", + "cardColor": "#b4ff00", + "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, - "min": 0, + "max": null, + "min": 1, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "Duration of unmarshal events from kv to SQL row", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 60 + "y": 30 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 101, + "id": 256, "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true + "show": true }, - "links": [], + "pluginVersion": "6.1.6", "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", "format": "heatmap", "instant": false, + "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "Mounter unmarshal duration", + "timeFrom": null, + "timeShift": null, + "title": "EtcdWorker exec etcd txn duration", "tooltip": { "show": true, "showHistogram": true @@ -6877,23 +10628,52 @@ "yBucketNumber": null, "yBucketSize": null }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "Percentiles of unmarshal events from kv to SQL row duration", - "fill": 0, +<<<<<<< HEAD + "description": "The size of batch resolved ts message from TiKV", +======= + "description": "", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, "fillGradient": 0, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 60 +<<<<<<< HEAD + "y": 67 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 97, +======= + "y": 30 }, "hiddenSeries": false, - "id": 55, + "id": 258, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, @@ -6907,172 +10687,94 @@ "total": false, "values": true }, - "lines": true, - "linewidth": 1, +<<<<<<< HEAD "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "reverseYBuckets": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{capture}}-p99", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", - "format": "time_series", - "hide": true, + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", + "format": "heatmap", "instant": false, - "intervalFactor": 1, - "legendFormat": "{{capture}}-p999", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Mounter unmarshal duration percentile", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of KV client dispatched event per second", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 67 - }, - "hiddenSeries": false, - "id": 31, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, + "intervalFactor": 2, + "legendFormat": "{{le}}", +======= "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, - "paceLength": 10, "percentage": false, + "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*batch-resolved/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_kvclient_send_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture, changefeed, type)", + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}-{{type}}", + "legendFormat": "{{capture}}-p95", + "queryType": "randomWalk", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { - "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture, changefeed, table)", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", + "hide": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}-batch-resolved", + "legendFormat": "{{capture}}-p99", "refId": "B" } ], +<<<<<<< HEAD + "title": "KV client batch resolved size", +======= "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV client dispatch events/s", + "title": "EtcdWorker exec etcd txn duration percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "show": true, + "showHistogram": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "none", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, +<<<<<<< HEAD + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null +======= "yaxes": [ { - "format": "short", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -7090,64 +10792,62 @@ "align": false, "alignLevel": null } +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "cards": { - "cardPadding": 0, - "cardRound": 0 + "cardPadding": null, + "cardRound": null }, "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolatePurples", "exponent": 0.5, "min": 0, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "The size of batch resolved ts message from TiKV", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 67 + "x": 0, + "y": 37 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 97, + "id": 254, "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true + "show": true }, - "links": [], + "pluginVersion": "6.1.6", "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", "format": "heatmap", "instant": false, - "intervalFactor": 2, + "interval": "", + "intervalFactor": 1, "legendFormat": "{{le}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "KV client batch resolved size", + "title": "EtcdWorker txn size", "tooltip": { "show": true, "showHistogram": true }, - "tooltipDecimals": 1, + "tooltipDecimals": null, "type": "heatmap", "xAxis": { "show": true @@ -7155,15 +10855,15 @@ "xBucketNumber": null, "xBucketSize": null, "yAxis": { - "decimals": 1, - "format": "none", + "decimals": null, + "format": "decbytes", "logBase": 1, "max": null, "min": null, "show": true, "splitFactor": null }, - "yBucketBound": "upper", + "yBucketBound": "auto", "yBucketNumber": null, "yBucketSize": null }, @@ -7173,17 +10873,33 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of regions that are being scanned", +======= + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, +<<<<<<< HEAD "x": 0, "y": 74 }, "hiddenSeries": false, "id": 177, +======= + "x": 12, + "y": 37 + }, + "hiddenSeries": false, + "id": 260, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, @@ -7199,13 +10915,18 @@ }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { +<<<<<<< HEAD "dataLinks": [] }, "paceLength": 10, +======= + "alertThreshold": true + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "percentage": false, + "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", @@ -7215,18 +10936,38 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_kvclient_region_token{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", +======= + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "interval": "", + "legendFormat": "{{capture}}-p95", + "queryType": "randomWalk", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "hide": false, + "interval": "", + "legendFormat": "{{capture}}-p99", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "KV client scanning regions", +======= + "title": "EtcdWorker txn size percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -7242,7 +10983,7 @@ }, "yaxes": [ { - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -7269,6 +11010,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Active stream count of each gRPC connection", "fill": 1, "fillGradient": 0, @@ -7280,6 +11022,20 @@ }, "hiddenSeries": false, "id": 188, +======= + "description": "The time consumed of writing WAL into the persistent storage in .99", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 291, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, @@ -7290,6 +11046,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -7297,12 +11054,15 @@ "linewidth": 1, "links": [], "nullPointMode": "null", +<<<<<<< HEAD "options": { "dataLinks": [] }, +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -7311,22 +11071,37 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_kvclient_grpc_stream_count{tidb_cluster=\"$tidb_cluster\"}) by (store)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{store}}", "refId": "A" +======= + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 4 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "KV client gRPC stream count", +======= + "title": "Etcd 99% WAL fsync duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -7338,7 +11113,11 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "short", +======= + "format": "s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -7365,6 +11144,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of regions that have not connected to TiKV", "fieldConfig": { "defaults": { @@ -7382,16 +11162,34 @@ }, "hiddenSeries": false, "id": 251, +======= + "description": "The time consumed of handling etcd transactions in .99", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 290, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, "current": true, +<<<<<<< HEAD "hideEmpty": true, "hideZero": true, +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -7399,12 +11197,15 @@ "linewidth": 1, "links": [], "nullPointMode": "null", +<<<<<<< HEAD "options": { "alertThreshold": true }, +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -7413,6 +11214,7 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "exemplar": true, "expr": "sum(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", "format": "time_series", @@ -7420,17 +11222,30 @@ "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", "refId": "A" +======= + "expr": "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, result, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{result}}", + "refId": "A", + "step": 4 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "KV client cached regions", +======= + "title": "Etcd 99% Handle transactions duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -7442,7 +11257,11 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "short", +======= + "format": "s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -7462,6 +11281,7 @@ "align": false, "alignLevel": null } +<<<<<<< HEAD }, { "aliasColors": {}, @@ -7569,6 +11389,11 @@ } ], "title": "Events", +======= + } + ], + "title": "Etcd", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "type": "row" }, { @@ -7592,7 +11417,7 @@ "h": 8, "w": 12, "x": 0, - "y": 5 + "y": 7 }, "id": 131, "legend": { @@ -7677,7 +11502,7 @@ "h": 8, "w": 12, "x": 12, - "y": 5 + "y": 7 }, "id": 132, "legend": { @@ -7762,7 +11587,11 @@ "h": 8, "w": 12, "x": 0, +<<<<<<< HEAD "y": 13 +======= + "y": 14 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 133, "legend": { @@ -7847,7 +11676,11 @@ "h": 8, "w": 12, "x": 12, +<<<<<<< HEAD "y": 13 +======= + "y": 14 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 134, "legend": { @@ -8078,7 +11911,11 @@ "h": 7, "w": 12, "x": 0, +<<<<<<< HEAD "y": 5 +======= + "y": 21 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 60, "legend": { @@ -8202,9 +12039,15 @@ "grid": {}, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 12, "x": 12, "y": 5 +======= + "w": 5, + "x": 7, + "y": 21 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 74, "legend": { @@ -8311,8 +12154,13 @@ "gridPos": { "h": 7, "w": 12, +<<<<<<< HEAD "x": 0, "y": 12 +======= + "x": 12, + "y": 21 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 147, "legend": { @@ -8396,6 +12244,88 @@ } }, { +<<<<<<< HEAD +======= + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The time consumed to CDC incremental scan", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 68, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "maxPerRow": 3, + "repeat": null, + "repeatDirection": "h", + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(tikv_cdc_scan_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Initial scan duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "aliasColors": {}, "bars": false, "dashLength": 10, @@ -8411,7 +12341,11 @@ "h": 7, "w": 12, "x": 12, +<<<<<<< HEAD "y": 12 +======= + "y": 28 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 194, "legend": { @@ -8573,9 +12507,15 @@ "grid": {}, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 7, "x": 0, "y": 19 +======= + "w": 6, + "x": 18, + "y": 28 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 152, "legend": { @@ -8704,9 +12644,15 @@ "grid": {}, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 5, "x": 7, "y": 19 +======= + "w": 12, + "x": 0, + "y": 35 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 153, "legend": { @@ -8802,7 +12748,11 @@ "h": 7, "w": 12, "x": 12, +<<<<<<< HEAD "y": 19 +======= + "y": 35 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 70, "legend": { @@ -8975,8 +12925,13 @@ "gridPos": { "h": 7, "w": 6, +<<<<<<< HEAD "x": 12, "y": 26 +======= + "x": 18, + "y": 35 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 72, "legend": { @@ -9069,9 +13024,15 @@ "fill": 1, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 6, "x": 18, "y": 26 +======= + "w": 12, + "x": 0, + "y": 42 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 140, "legend": { @@ -9182,8 +13143,13 @@ "gridPos": { "h": 7, "w": 12, +<<<<<<< HEAD "x": 0, "y": 33 +======= + "x": 12, + "y": 42 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 78, "legend": { @@ -9289,9 +13255,15 @@ "grid": {}, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 6, "x": 12, "y": 33 +======= + "w": 12, + "x": 0, + "y": 49 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 76, "legend": { @@ -9391,8 +13363,13 @@ "gridPos": { "h": 7, "w": 6, +<<<<<<< HEAD "x": 18, "y": 33 +======= + "x": 12, + "y": 49 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 139, "legend": { @@ -9488,9 +13465,15 @@ "fill": 4, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 12, "x": 0, "y": 40 +======= + "w": 6, + "x": 18, + "y": 49 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 143, "legend": { @@ -9645,6 +13628,23 @@ "pointradius": 1, "points": false, "renderer": "flot", +<<<<<<< HEAD +======= + "repeat": null, + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "172.16.5.33:47912", + "value": "172.16.5.33:47912" + }, + "runtime_instance": { + "selected": false, + "text": "172.16.5.37:47912", + "value": "172.16.5.37:47912" + } + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "seriesOverrides": [ { "alias": "/.*len/", @@ -9766,6 +13766,23 @@ "pointradius": 5, "points": false, "renderer": "flot", +<<<<<<< HEAD +======= + "repeat": null, + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "172.16.5.33:47912", + "value": "172.16.5.33:47912" + }, + "runtime_instance": { + "selected": false, + "text": "172.16.5.37:47912", + "value": "172.16.5.37:47912" + } + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -9942,7 +13959,25 @@ "pointradius": 5, "points": false, "renderer": "flot", +<<<<<<< HEAD "seriesOverrides": [], +======= + "scopedVars": { + "instance": { + "selected": false, + "text": "172.16.5.33:47912", + "value": "172.16.5.33:47912" + }, + "runtime_instance": { + "selected": false, + "text": "172.16.5.37:47912", + "value": "172.16.5.37:47912" + } + }, + "seriesOverrides": [ + {} + ], +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "spaceLength": 10, "stack": false, "steppedLine": false, @@ -10077,8 +14112,8 @@ }, "runtime_instance": { "selected": false, - "text": "172.16.5.32:47800", - "value": "172.16.5.32:47800" + "text": "172.16.5.37:47912", + "value": "172.16.5.37:47912" } }, "seriesOverrides": [ @@ -10794,5 +14829,10 @@ "timezone": "browser", "title": "Test-Cluster-TiCDC", "uid": "YiGL8hBZ1", +<<<<<<< HEAD "version": 26 } +======= + "version": 29 +} +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038))