From 728b7a899d83527d433ce0fa234d8c2c49be953d Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Tue, 28 Dec 2021 14:37:49 +0800 Subject: [PATCH 1/3] This is an automated cherry-pick of #4038 Signed-off-by: ti-chi-bot --- cdc/owner/changefeed.go | 34 +- cdc/owner/metrics.go | 18 +- metrics/grafana/ticdc.json | 6216 +++++++++++++++++++++++++++++------- 3 files changed, 5176 insertions(+), 1092 deletions(-) diff --git a/cdc/owner/changefeed.go b/cdc/owner/changefeed.go index 6f9071ed54c..4e5b1d7c59f 100644 --- a/cdc/owner/changefeed.go +++ b/cdc/owner/changefeed.go @@ -64,6 +64,8 @@ type changefeed struct { metricsChangefeedCheckpointTsGauge prometheus.Gauge metricsChangefeedCheckpointTsLagGauge prometheus.Gauge + metricsChangefeedResolvedTsGauge prometheus.Gauge + metricsChangefeedResolvedTsLagGauge prometheus.Gauge newDDLPuller func(ctx cdcContext.Context, startTs uint64) (DDLPuller, error) newSink func(ctx cdcContext.Context) (AsyncSink, error) @@ -258,6 +260,18 @@ LOOP: // init metrics c.metricsChangefeedCheckpointTsGauge = changefeedCheckpointTsGauge.WithLabelValues(c.id) c.metricsChangefeedCheckpointTsLagGauge = changefeedCheckpointTsLagGauge.WithLabelValues(c.id) +<<<<<<< HEAD +======= + c.metricsChangefeedResolvedTsGauge = changefeedResolvedTsGauge.WithLabelValues(c.id) + c.metricsChangefeedResolvedTsLagGauge = changefeedResolvedTsLagGauge.WithLabelValues(c.id) + + // create scheduler + c.scheduler, err = c.newScheduler(ctx, checkpointTs) + if err != nil { + return errors.Trace(err) + } + +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) c.initialized = true return nil } @@ -279,10 +293,21 @@ func (c *changefeed) releaseResources() { log.Warn("Closing sink failed in Owner", zap.String("changefeedID", c.state.ID), zap.Error(err)) } c.wg.Wait() +<<<<<<< HEAD +======= + c.scheduler.Close(ctx) + +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) changefeedCheckpointTsGauge.DeleteLabelValues(c.id) changefeedCheckpointTsLagGauge.DeleteLabelValues(c.id) c.metricsChangefeedCheckpointTsGauge = nil c.metricsChangefeedCheckpointTsLagGauge = nil + + changefeedResolvedTsGauge.DeleteLabelValues(c.id) + changefeedResolvedTsLagGauge.DeleteLabelValues(c.id) + c.metricsChangefeedResolvedTsGauge = nil + c.metricsChangefeedResolvedTsLagGauge = nil + c.initialized = false } @@ -461,10 +486,13 @@ func (c *changefeed) updateStatus(currentTs int64, barrierTs model.Ts) { } return status, changed, nil }) - phyTs := oracle.ExtractPhysical(checkpointTs) + phyCkpTs := oracle.ExtractPhysical(checkpointTs) + c.metricsChangefeedCheckpointTsGauge.Set(float64(phyCkpTs)) + c.metricsChangefeedCheckpointTsLagGauge.Set(float64(currentTs-phyCkpTs) / 1e3) - c.metricsChangefeedCheckpointTsGauge.Set(float64(phyTs)) - c.metricsChangefeedCheckpointTsLagGauge.Set(float64(currentTs-phyTs) / 1e3) + phyRTs := oracle.ExtractPhysical(resolvedTs) + c.metricsChangefeedResolvedTsGauge.Set(float64(phyRTs)) + c.metricsChangefeedResolvedTsLagGauge.Set(float64(currentTs-phyRTs) / 1e3) } func (c *changefeed) Close() { diff --git a/cdc/owner/metrics.go b/cdc/owner/metrics.go index 4aeb513649b..bc1bff8d0aa 100644 --- a/cdc/owner/metrics.go +++ b/cdc/owner/metrics.go @@ -28,7 +28,21 @@ var ( Namespace: "ticdc", Subsystem: "owner", Name: "checkpoint_ts_lag", - Help: "checkpoint ts lag of changefeeds", + Help: "checkpoint ts lag of changefeeds in seconds", + }, []string{"changefeed"}) + changefeedResolvedTsGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "owner", + Name: "resolved_ts", + Help: "resolved ts of changefeeds", + }, []string{"changefeed"}) + changefeedResolvedTsLagGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "owner", + Name: "resolved_ts_lag", + Help: "resolved ts lag of changefeeds in seconds", }, []string{"changefeed"}) ownershipCounter = prometheus.NewCounter( prometheus.CounterOpts{ @@ -63,7 +77,9 @@ const ( // InitMetrics registers all metrics used in owner func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(changefeedCheckpointTsGauge) + registry.MustRegister(changefeedResolvedTsGauge) registry.MustRegister(changefeedCheckpointTsLagGauge) + registry.MustRegister(changefeedResolvedTsLagGauge) registry.MustRegister(ownershipCounter) registry.MustRegister(ownerMaintainTableNumGauge) registry.MustRegister(changefeedStatusGauge) diff --git a/metrics/grafana/ticdc.json b/metrics/grafana/ticdc.json index f0e0d123f4f..d992b807725 100644 --- a/metrics/grafana/ticdc.json +++ b/metrics/grafana/ticdc.json @@ -125,7 +125,11 @@ "gnetId": null, "graphTooltip": 1, "id": null, +<<<<<<< HEAD "iteration": 1636101093950, +======= + "iteration": 1640447119260, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "links": [], "panels": [ { @@ -196,6 +200,13 @@ "intervalFactor": 1, "legendFormat": "TiKV-{{instance}}", "refId": "B" + }, + { + "expr": "(time() - process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", job=\"pd\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "PD-{{instance}}", + "refId": "C" } ], "thresholds": [], @@ -823,24 +834,22 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The history of PD cluster leadership, leader node has a value that is great than 0", "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 8 }, - "hiddenSeries": false, - "id": 114, + "id": 293, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "max": true, + "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -849,10 +858,6 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -860,35 +865,23 @@ "seriesOverrides": [], "spaceLength": 10, "stack": false, - "steppedLine": false, + "steppedLine": true, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", + "expr": "pd_tso_role{tidb_cluster=\"$tidb_cluster\", dc=\"global\"} > BOOL 0.5", "format": "time_series", + "hide": false, + "interval": "30s", "intervalFactor": 1, - "legendFormat": "p999-{{instance}}", + "legendFormat": "PD-{{instance}}", "refId": "A" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p99-{{instance}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p95-{{instance}}", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Etcd health check duration", + "title": "PD leader history", "tooltip": { "shared": true, "sort": 0, @@ -904,7 +897,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -938,9 +931,10 @@ "x": 0, "y": 1 }, - "id": 266, + "id": 11, "panels": [ { +<<<<<<< HEAD "cards": { "cardPadding": 1, "cardRound": null @@ -954,29 +948,71 @@ "mode": "spectrum" }, "dataFormat": "tsbuckets", +======= + "columns": [ + { + "text": "Current", + "value": "current" + } + ], +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "The number of captured table of TiCDC nodes ", + "fontSize": "100%", "gridPos": { +<<<<<<< HEAD "h": 8, "w": 12, +======= + "h": 5, + "w": 7, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "x": 0, - "y": 2 + "y": 3 }, +<<<<<<< HEAD "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, "id": 262, "legend": { "show": true +======= + "id": 4, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], "targets": [ { +<<<<<<< HEAD "exemplar": true, "expr": "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, capture)", "format": "heatmap", @@ -1011,21 +1047,34 @@ "yBucketBound": "auto", "yBucketNumber": null, "yBucketSize": null +======= + "expr": "sum(ticdc_processor_num_of_tables{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{capture}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Changefeed table count", + "transform": "timeseries_aggregations", + "type": "table" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "columns": [ + { + "text": "Current", + "value": "current" + } + ], "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, + "description": "Internal resolved ts of TiCDC nodes", + "fontSize": "100%", "gridPos": { +<<<<<<< HEAD "h": 8, "w": 12, "x": 12, @@ -1041,60 +1090,107 @@ "show": true, "total": false, "values": false +======= + "h": 10, + "w": 7, + "x": 7, + "y": 3 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "id": 90, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true }, - "percentage": false, - "pluginVersion": "6.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "styles": [ + { + "alias": "table", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Metric", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "resolved ts", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "MM-DD HH:mm:ss.SSS", + "decimals": 2, + "pattern": "Current", + "thresholds": [], + "type": "date", + "unit": "short" + } + ], "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "max(ticdc_processor_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed)", + "format": "time_series", "interval": "", - "legendFormat": "{{capture}}-95", - "queryType": "randomWalk", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{changefeed}}", "refId": "A" }, { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, + "expr": "max(ticdc_processor_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed) > 0", + "format": "time_series", "interval": "", - "legendFormat": "{{capture}}-99", + "intervalFactor": 1, + "legendFormat": "checkpoint-{{capture}}-{{changefeed}}", "refId": "B" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "EtcdWorker tick reactor duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "title": "Processor resolved ts", + "transform": "timeseries_aggregations", + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "${DS_TEST-CLUSTER}", + "description": "Internal resolved ts of captured tables", + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 10, + "x": 14, + "y": 3 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "id": 30, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true }, - "yaxes": [ + "styles": [ { +<<<<<<< HEAD "$$hashKey": "object:2612", "format": "s", "label": null, @@ -1111,142 +1207,203 @@ "max": null, "min": null, "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": 1, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", +======= + "alias": "table", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Metric", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "resolved ts", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "MM-DD HH:mm:ss.SSS", + "decimals": 2, + "pattern": "Current", + "thresholds": [], + "type": "date", + "unit": "short" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + } + ], + "targets": [ + { + "expr": "bottomk(10, max(ticdc_processor_table_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed,table))", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{changefeed}}-{{table}}", + "refId": "A" + }, + { + "expr": "bottomk(10, max(ticdc_processor_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed,table) > 0)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "checkpoint-{{capture}}-{{changefeed}}", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Table resolved ts", + "transform": "timeseries_aggregations", + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "The number of replicated tables maintained in owner", + "fontSize": "100%", "gridPos": { +<<<<<<< HEAD "h": 8, "w": 12, "x": 0, "y": 10 +======= + "h": 5, + "w": 7, + "x": 0, + "y": 8 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 256, - "legend": { - "show": true + "id": 138, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, + "expr": "sum(ticdc_owner_maintain_table_num{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\",type=\"total\"}) by (capture)", + "format": "time_series", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "queryType": "randomWalk", + "intervalFactor": 1, + "legendFormat": "{{capture}}-total", "refId": "A" + }, + { + "expr": "sum(ticdc_owner_maintain_table_num{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\",type=\"wip\"}) by (capture)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{capture}}-wip", + "refId": "B" } ], "timeFrom": null, "timeShift": null, - "title": "EtcdWorker exec etcd txn duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "title": "Table count maintained by owner", + "transform": "timeseries_aggregations", + "type": "table" }, { "aliasColors": {}, - "bars": false, + "bars": true, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, + "description": "The checkpoint ts of changefeeds.", + "fill": 0, "gridPos": { +<<<<<<< HEAD "h": 8, "w": 12, "x": 12, "y": 10 +======= + "h": 7, + "w": 9, + "x": 0, + "y": 13 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "hiddenSeries": false, - "id": 258, + "id": 86, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, "total": false, - "values": false + "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, + "links": [], "nullPointMode": "null", - "options": { - "alertThreshold": true - }, "percentage": false, - "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/approximate current time.*/", + "bars": false + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "max(pd_cluster_tso{tidb_cluster=\"$tidb_cluster\"})", "format": "time_series", + "hide": false, "interval": "", +<<<<<<< HEAD "legendFormat": "{{capture}}-p95", "queryType": "randomWalk", "refId": "A" @@ -1257,14 +1414,35 @@ "hide": false, "interval": "", "legendFormat": "{{lcapture}-p99}", +======= + "intervalFactor": 1, + "legendFormat": "approximate current time (s)", + "refId": "A" + }, + { + "expr": "max(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "B" + }, + { + "expr": "max(ticdc_owner_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed) > 0", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-resolvedts", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker exec etcd txn duration percentile", + "title": "Changefeed checkpoint", "tooltip": { "shared": true, "sort": 0, @@ -1273,6 +1451,7 @@ "type": "graph", "xaxis": { "buckets": null, + "max": null, "mode": "time", "name": null, "show": true, @@ -1280,10 +1459,14 @@ }, "yaxes": [ { +<<<<<<< HEAD "$$hashKey": "object:1612", "format": "s", +======= + "format": "dateTimeAsIso", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -1295,7 +1478,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -1304,77 +1487,108 @@ } }, { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolatePurples", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "Request count of etcd operation per second", + "fill": 1, + "fillGradient": 0, "gridPos": { +<<<<<<< HEAD "h": 8, "w": 12, "x": 0, "y": 18 +======= + "h": 7, + "w": 8, + "x": 9, + "y": 13 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 254, + "hiddenSeries": false, + "id": 102, "legend": { - "show": true + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, capture)", - "format": "heatmap", - "instant": false, + "expr": "sum(rate(ticdc_etcd_request_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (capture, type)", + "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{le}}", - "queryType": "randomWalk", + "legendFormat": "{{capture}}-{{type}}", "refId": "A" } ], - "title": "EtcdWorker txn size ", + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "PD etcd requests/s", "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": null, - "type": "heatmap", - "xAxis": { - "show": true + "shared": true, + "sort": 0, + "value_type": "individual" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "decbytes", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -1382,38 +1596,38 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "The number of errors that interrupt changefeed per minute ", "fill": 1, - "fillGradient": 0, "gridPos": { +<<<<<<< HEAD "h": 8, "w": 12, "x": 12, "y": 18 +======= + "h": 7, + "w": 7, + "x": 17, + "y": 13 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "hiddenSeries": false, - "id": 260, + "id": 82, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", - "options": { - "alertThreshold": true - }, "percentage": false, - "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", @@ -1423,27 +1637,18 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "interval": "", - "legendFormat": "{{capture}}-p95", - "queryType": "randomWalk", + "expr": "sum(delta(ticdc_processor_exit_with_error_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, - "interval": "", - "legendFormat": "{{capture}}-p99", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker txn size percentile", + "title": "Exit error count/m", "tooltip": { "shared": true, "sort": 0, @@ -1459,8 +1664,12 @@ }, "yaxes": [ { +<<<<<<< HEAD "$$hashKey": "object:2055", "format": "bytes", +======= + "format": "short", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -1481,6 +1690,7 @@ "align": false, "alignLevel": null } +<<<<<<< HEAD } ], "title": "EtcdWorker", @@ -1606,6 +1816,46 @@ }, "percentage": false, "pluginVersion": "6.1.6", +======= + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "pointradius": 2, "points": false, "renderer": "flot", @@ -1615,6 +1865,7 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", "interval": "", @@ -1628,14 +1879,44 @@ "hide": false, "interval": "", "legendFormat": "{{capture}}-99", +======= + "expr": "max(ticdc_owner_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}", + "refId": "A" + }, + { + "expr": "sum(ticdc_processor_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture,changefeed)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", "refId": "B" + }, + { + "expr": "max(ticdc_owner_resolved_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-resolvedts", + "refId": "C" + }, + { + "expr": "sum(ticdc_processor_resolved_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture,changefeed)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}-resolvedts", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker tick reactor duration", + "title": "Changefeed checkpoint lag", "tooltip": { "shared": true, "sort": 0, @@ -1651,17 +1932,15 @@ }, "yaxes": [ { - "$$hashKey": "object:2612", "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "$$hashKey": "object:2613", - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -1675,122 +1954,45 @@ } }, { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": 1, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "The derivative of a changefeed checkpoint, ideally it should not be 0 (means the changefeed is stopped).", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 256, - "legend": { - "show": true - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "EtcdWorker exec etcd txn duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 10 + "y": 20 }, "hiddenSeries": false, - "id": 258, + "id": 2, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, + "hideEmpty": false, + "hideZero": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { - "alertThreshold": true + "dataLinks": [] }, + "paceLength": 10, "percentage": false, - "pluginVersion": "6.1.6", - "pointradius": 2, + "pointradius": 0.5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -1799,28 +2001,19 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "sum(deriv(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed) / 1000 > 0", "format": "time_series", "interval": "", - "legendFormat": "{{capture}}-p95", - "queryType": "randomWalk", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, - "interval": "", - "legendFormat": "{{capture}-p99}", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker exec etcd txn duration percentile", + "title": "Changefeed checkpoint derivative", "tooltip": { "shared": true, "sort": 0, @@ -1836,16 +2029,14 @@ }, "yaxes": [ { - "$$hashKey": "object:1612", "format": "s", "label": null, "logBase": 2, - "max": null, - "min": null, + "max": "3600", + "min": "0.001", "show": true }, { - "$$hashKey": "object:1613", "format": "short", "label": null, "logBase": 1, @@ -1859,119 +2050,38 @@ "alignLevel": null } }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolatePurples", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 254, - "legend": { - "show": true - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, capture)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "EtcdWorker txn size ", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": null, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "decbytes", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "The status of each changefeed.\n\n0: Normal\n\n1: Error\n\n2: Failed\n\n3: Stopped\n\n4: Finished\n\n-1: Unknown", "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 12, - "y": 18 + "x": 0, + "y": 27 }, - "hiddenSeries": false, - "id": 260, + "id": 163, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, - "min": false, + "current": true, + "max": true, + "min": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", - "options": { - "alertThreshold": true - }, "percentage": false, - "pluginVersion": "6.1.6", - "pointradius": 2, - "points": false, + "pointradius": 1, + "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -1979,27 +2089,19 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "interval": "", - "legendFormat": "{{capture}}-p95", - "queryType": "randomWalk", + "expr": "ticdc_owner_status{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{changefeed}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, - "interval": "", - "legendFormat": "{{capture}}-p99", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker txn size percentile", + "title": "The status of changefeeds", "tooltip": { "shared": true, "sort": 0, @@ -2015,8 +2117,7 @@ }, "yaxes": [ { - "$$hashKey": "object:2055", - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2024,7 +2125,6 @@ "show": true }, { - "$$hashKey": "object:2056", "format": "short", "label": null, "logBase": 1, @@ -2037,36 +2137,1084 @@ "align": false, "alignLevel": null } - } - ], - "title": "EtcdWorker", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 11, - "panels": [ + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Estimate the remaining time for a changefeed catch-up upstream TiDB.", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 27 + }, + "hiddenSeries": false, + "id": 253, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "abs(max(ticdc_owner_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"} / (deriv(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])/1000)) by (changefeed))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Changefeed checkpoint catch-up ETA", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "", + "logBase": 2, + "max": "604800", + "min": "1", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 1, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "Sink write duration of changefeeds", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 34 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 94, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "repeat": null, + "repeatDirection": "h", + "reverseYBuckets": false, + "targets": [ + { + "expr": "max(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Sink write duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Percentiles of sink write duration of changefeeds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 34 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p999", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sink write duration percentile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The number of changed rows that are written to downstream per second", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate(ticdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}", + "refId": "A" + }, + { + "expr": "sum (rate(ticdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, +<<<<<<< HEAD + "title": "EtcdWorker tick reactor duration", +======= + "title": "Sink write rows count/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { +<<<<<<< HEAD + "$$hashKey": "object:2612", + "format": "s", +======= + "format": "none", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2613", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { +<<<<<<< HEAD + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 1, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 256, + "legend": { + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "EtcdWorker exec etcd txn duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "hiddenSeries": false, + "id": 258, + "legend": { + "avg": false, + "current": false, +======= + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Percentiles of sink batch size", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 41 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "6.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { +<<<<<<< HEAD + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "interval": "", + "legendFormat": "{{capture}}-p95", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "hide": false, + "interval": "", + "legendFormat": "{{capture}-p99}", + "refId": "B" +======= + "expr": "histogram_quantile(0.90, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-p90", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-p99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p999", + "refId": "C" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, +<<<<<<< HEAD + "title": "EtcdWorker exec etcd txn duration percentile", +======= + "title": "Sink write batch size percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { +<<<<<<< HEAD + "$$hashKey": "object:1612", + "format": "s", +======= + "format": "none", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1613", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { +<<<<<<< HEAD + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolatePurples", + "exponent": 0.5, +======= + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 +======= + "description": "Asynchronous flush sink duration of changefeeds", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 48 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, +<<<<<<< HEAD + "id": 254, + "legend": { + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, capture)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "EtcdWorker txn size ", +======= + "id": 93, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "expr": "max(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Flush sink duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "show": true, + "showHistogram": true + }, +<<<<<<< HEAD + "tooltipDecimals": null, +======= + "tooltipDecimals": 1, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { +<<<<<<< HEAD + "decimals": null, + "format": "decbytes", +======= + "decimals": 1, + "format": "s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, +<<<<<<< HEAD + "yBucketBound": "auto", +======= + "yBucketBound": "upper", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 260, +======= + "description": "Percentiles of asynchronous flush sink duration of changefeeds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 48 + }, + "hiddenSeries": false, + "id": 98, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { +<<<<<<< HEAD + "alertThreshold": true + }, +======= + "dataLinks": [] + }, + "paceLength": 10, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "percentage": false, + "pluginVersion": "6.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { +<<<<<<< HEAD + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "interval": "", + "legendFormat": "{{capture}}-p95", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "hide": false, + "interval": "", + "legendFormat": "{{capture}}-p99", + "refId": "B" +======= + "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p999", + "refId": "C" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, +<<<<<<< HEAD + "title": "EtcdWorker txn size percentile", +======= + "title": "Flush sink duration percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { +<<<<<<< HEAD + "$$hashKey": "object:2055", + "format": "bytes", +======= + "format": "s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2056", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } +<<<<<<< HEAD + } + ], + "title": "EtcdWorker", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 11, + "panels": [ { "columns": [ { - "text": "Current", - "value": "current" + "text": "Current", + "value": "current" + } + ], + "datasource": "${DS_TEST-CLUSTER}", + "description": "The number of captured table of TiCDC nodes ", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 2 +======= + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 1, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The duration of detecting and waiting conflict of MySQL sink", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 55 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 103, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "repeatDirection": "h", + "reverseYBuckets": false, + "targets": [ + { + "expr": "max(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" } ], + "title": "MySQL sink conflict detect duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of captured table of TiCDC nodes ", - "fontSize": "100%", + "description": "Distribution of MySQL worker loads", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 5, - "w": 7, - "x": 0, - "y": 2 + "h": 7, + "w": 12, + "x": 12, + "y": 55 + }, + "hiddenSeries": false, + "id": 95, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 4, "links": [], @@ -2077,6 +3225,7 @@ "col": 0, "desc": true }, +<<<<<<< HEAD "styles": [ { "alias": "Time", @@ -2102,34 +3251,236 @@ "targets": [ { "expr": "sum(ticdc_processor_num_of_tables{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", +======= + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture,bucket)", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{bucket}}", + "refId": "A" + }, + { + "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) >= 0)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", + "hide": true, "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{capture}}", "refId": "A" +======= + "legendFormat": "total worker", + "refId": "B" + }, + { + "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) <= 2)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "0-2 row/s worker", + "refId": "C" + }, + { + "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) > 2 and rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) <= 10)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "2-10 row/s worker", + "refId": "D" + }, + { + "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) > 10 and rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) <= 100)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "10-100 row/s worker", + "refId": "E" + }, + { + "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) > 100)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ">100 row/s worker", + "refId": "F" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "timeFrom": null, "timeShift": null, +<<<<<<< HEAD "title": "Changefeed table count", "transform": "timeseries_aggregations", "type": "table" }, { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], + "columns": [ +======= + "title": "MySQL sink worker load", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + { + "text": "Current", + "value": "current" + } + ], +<<<<<<< HEAD + "datasource": "${DS_TEST-CLUSTER}", + "description": "Internal resolved ts of TiCDC nodes", + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 7, + "x": 7, + "y": 2 +======= + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 1, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The duration of executing DDL in MySQL sink", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 62 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 165, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "repeatDirection": "h", + "reverseYBuckets": false, + "targets": [ + { + "expr": "max(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "MySQL sink ddl execution duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Internal resolved ts of TiCDC nodes", - "fontSize": "100%", + "description": "Percentiles of detecting and waiting conflict duration of MySQL sink", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 10, - "w": 7, - "x": 7, - "y": 2 + "h": 7, + "w": 12, + "x": 12, + "y": 62 + }, + "hiddenSeries": false, + "id": 83, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 90, "links": [], @@ -2140,6 +3491,7 @@ "col": 0, "desc": true }, +<<<<<<< HEAD "styles": [ { "alias": "table", @@ -2189,22 +3541,83 @@ "intervalFactor": 1, "legendFormat": "checkpoint-{{capture}}-{{changefeed}}", "refId": "B" +======= + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.999,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p999", + "refId": "C" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "timeFrom": null, "timeShift": null, +<<<<<<< HEAD "title": "Processor resolved ts", "transform": "timeseries_aggregations", "type": "table" }, { "columns": [ +======= + "title": "MySQL sink conflict detect duration percentile", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) { "text": "Current", "value": "current" } ], "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Internal resolved ts of captured tables", "fontSize": "100%", "gridPos": { @@ -2212,18 +3625,75 @@ "w": 10, "x": 14, "y": 2 +======= + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 149, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 30, "links": [], +<<<<<<< HEAD "pageSize": null, "scroll": true, "showHeader": true, "sort": { "col": 0, "desc": true +======= + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ capture }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Processor Memory Consumption Per Capture", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "styles": [ { +<<<<<<< HEAD "alias": "table", "colorMode": null, "colors": [ @@ -2238,6 +3708,14 @@ "thresholds": [], "type": "string", "unit": "short" +======= + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "alias": "resolved ts", @@ -2255,13 +3733,68 @@ "unit": "short" } ], +<<<<<<< HEAD "targets": [ { "expr": "bottomk(10, max(ticdc_processor_table_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed,table))", +======= + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Percentiles of executing DDL duration of MySQL sink", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 69 + }, + "hiddenSeries": false, + "id": 166, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{capture}}-{{changefeed}}-{{table}}", "refId": "A" }, @@ -2272,10 +3805,31 @@ "intervalFactor": 1, "legendFormat": "checkpoint-{{capture}}-{{changefeed}}", "refId": "B" +======= + "legendFormat": "{{instance}}-p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p99", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.999,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p999", + "refId": "C" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "timeFrom": null, "timeShift": null, +<<<<<<< HEAD "title": "Table resolved ts", "transform": "timeseries_aggregations", "type": "table" @@ -2347,11 +3901,49 @@ "refId": "B" } ], - "timeFrom": null, - "timeShift": null, - "title": "Table count maintained by owner", - "transform": "timeseries_aggregations", - "type": "table" + "timeFrom": null, + "timeShift": null, + "title": "Table count maintained by owner", + "transform": "timeseries_aggregations", + "type": "table" +======= + "title": "MySQL sink ddl execution duration percentile", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "aliasColors": {}, @@ -2360,6 +3952,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The checkpoint ts of changefeeds.", "fill": 0, "gridPos": { @@ -2369,17 +3962,29 @@ "y": 12 }, "id": 86, +======= + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 164, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, "avg": false, - "current": true, + "current": false, "max": false, "min": false, +<<<<<<< HEAD "rightSide": false, +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, "sideWidth": null, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 2, @@ -2400,10 +4005,15 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "max(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed) > 0", +======= + "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture, changefeed)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{changefeed}}", "refId": "A" }, @@ -2415,13 +4025,21 @@ "intervalFactor": 1, "legendFormat": "approximate current time (s)", "refId": "B" +======= + "legendFormat": "{{ capture }}-{{ changefeed }}", + "refId": "A" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Changefeed checkpoint", +======= + "title": "Processor Memory Consumption Per Changefeed", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -2438,7 +4056,11 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "dateTimeAsIso", +======= + "format": "bytes", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -2459,12 +4081,180 @@ "alignLevel": null } }, + { + "columns": [], + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dateTimeAsSystem" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "capture" + }, + "properties": [ + { + "id": "custom.width", + "value": 191 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "changefeed" + }, + "properties": [ + { + "id": "custom.width", + "value": 209 + } + ] + } + ] + }, + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 200, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "last-snap-ts" + } + ] + }, + "pageSize": null, + "pluginVersion": "6.1.6", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Value", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "exemplar": true, + "expr": "ticdc_processor_schema_storage_gc_ts{changefeed=~\"$changefeed\", capture=~\"$capture\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Schema Storage GC progress", + "transform": "table", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "capture", + "changefeed", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value #A": "last-snap-ts" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "last-snap-ts" + } + ] + } + } + ], + "type": "table" + } + ], + "title": "Changefeed", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 206, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Request count of etcd operation per second", "fill": 1, "fillGradient": 0, @@ -2478,13 +4268,28 @@ "id": 102, "legend": { "alignAsTable": true, +======= + "description": "The number of events that puller outputs to sorter \n per second", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "hiddenSeries": false, + "id": 218, + "legend": { + "alignAsTable": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -2507,11 +4312,18 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(rate(ticdc_etcd_request_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (capture, type)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{capture}}-{{type}}", +======= + "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -2519,7 +4331,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "PD etcd requests/s", +======= + "title": "Puller output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -2548,7 +4364,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -2562,22 +4378,36 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of errors that interrupt changefeed per minute ", +======= + "description": "The total number of events that puller outputs", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "gridPos": { +<<<<<<< HEAD "h": 7, "w": 7, "x": 17, "y": 12 }, "id": 82, +======= + "h": 4, + "w": 12, + "x": 12, + "y": 4 + }, + "hiddenSeries": false, + "id": 229, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, - "max": false, + "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -2596,10 +4426,17 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(delta(ticdc_processor_exit_with_error_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{capture}}", +======= + "expr": "sum(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -2607,7 +4444,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Exit error count/m", +======= + "title": "Puller output events", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -2645,6 +4486,7 @@ } }, { +<<<<<<< HEAD "aliasColors": {}, "bars": false, "dashLength": 10, @@ -2748,31 +4590,49 @@ } }, { +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The derivative of a changefeed checkpoint, ideally it should not be 0 (means the changefeed is stopped).", +======= + "description": "The number of events that sorter outputs to puller \n per second", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 4, "w": 12, +<<<<<<< HEAD "x": 12, "y": 19 }, "hiddenSeries": false, "id": 2, +======= + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 228, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, +<<<<<<< HEAD "hideEmpty": false, "hideZero": false, "max": false, +======= + "max": true, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -2795,11 +4655,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(deriv(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed) / 1000 > 0", +======= + "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture, type)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{changefeed}}", +======= + "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -2807,7 +4675,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Changefeed checkpoint derivative", +======= + "title": "Sorter output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -2823,6 +4695,7 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "s", "label": null, "logBase": 2, @@ -2832,11 +4705,15 @@ }, { "format": "short", +======= + "format": "none", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, "min": null, "show": true +<<<<<<< HEAD } ], "yaxis": { @@ -2917,6 +4794,8 @@ "max": null, "min": null, "show": true +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "format": "short", @@ -2924,7 +4803,11 @@ "logBase": 1, "max": null, "min": null, +<<<<<<< HEAD "show": true +======= + "show": false +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "yaxis": { @@ -2938,6 +4821,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Estimate the remaining time for a changefeed catch-up upstream TiDB.", "fieldConfig": { "defaults": { @@ -2945,18 +4829,30 @@ }, "overrides": [] }, +======= + "description": "The total number of events that sorter outputs", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 12, +<<<<<<< HEAD "y": 26 }, "hiddenSeries": false, "id": 253, "legend": { - "alignAsTable": true, + "alignAsTable": true, +======= + "y": 8 + }, + "hiddenSeries": false, + "id": 220, + "legend": { + "alignAsTable": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "avg": false, "current": true, "hideEmpty": true, @@ -2986,12 +4882,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "exemplar": true, "expr": "abs(max(ticdc_owner_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"} / (deriv(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])/1000)) by (changefeed))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{changefeed}}", +======= + "expr": "sum(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -2999,7 +4902,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Changefeed checkpoint catch-up ETA", +======= + "title": "Sorter output events", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -3015,11 +4922,19 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "s", "label": "", "logBase": 2, "max": "604800", "min": "1", +======= + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true }, { @@ -3037,46 +4952,50 @@ } }, { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": 1, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Sink write duration of changefeeds", +======= + "description": "The number of events that mounter outputs to sink per second", + "fill": 1, + "fillGradient": 0, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 0, +<<<<<<< HEAD "y": 33 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, "id": 94, +======= + "y": 12 + }, + "hiddenSeries": false, + "id": 219, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, + "lines": true, + "linewidth": 1, "links": [], +<<<<<<< HEAD "repeat": null, "repeatDirection": "h", "reverseYBuckets": false, @@ -3091,29 +5010,70 @@ } ], "title": "Sink write duration", - "tooltip": { - "show": true, - "showHistogram": true +======= + "nullPointMode": "null", + "options": { + "dataLinks": [] }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mounter output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -3121,19 +5081,30 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Percentiles of sink write duration of changefeeds", +======= + "description": "The total number of events that mounter outputs", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 12, +<<<<<<< HEAD "y": 33 }, "hiddenSeries": false, "id": 35, +======= + "y": 12 + }, + "hiddenSeries": false, + "id": 224, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, "max": false, @@ -3161,11 +5132,16 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", +======= + "expr": "sum(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{instance}}-p95", + "legendFormat": "{{changefeed}}-{{capture}}", "refId": "A" +<<<<<<< HEAD }, { "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", @@ -3180,13 +5156,19 @@ "intervalFactor": 1, "legendFormat": "{{instance}}-p999", "refId": "C" +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Sink write duration percentile", +======= + "title": "Mounter output events", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -3202,9 +5184,9 @@ }, "yaxes": [ { - "format": "s", + "format": "none", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -3229,6 +5211,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of changed rows that are written to downstream per second", "fill": 1, "fillGradient": 0, @@ -3350,6 +5333,26 @@ "max": false, "min": false, "rightSide": true, +======= + "description": "The number of events that table sorter outputs to buffer sink per second", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 223, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, "total": false, "values": true @@ -3372,10 +5375,17 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "histogram_quantile(0.90, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{capture}}-p90", +======= + "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { @@ -3398,7 +5408,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Sink write batch size percentile", +======= + "title": "Table sink output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -3427,7 +5441,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -3518,19 +5532,30 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Percentiles of asynchronous flush sink duration of changefeeds", +======= + "description": "The total number of events that table sinks emit", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 12, +<<<<<<< HEAD "y": 47 }, "hiddenSeries": false, "id": 98, +======= + "y": 16 + }, + "hiddenSeries": false, + "id": 221, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, "max": false, @@ -3558,6 +5583,7 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", "format": "time_series", "intervalFactor": 1, @@ -3577,13 +5603,24 @@ "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}-p999", "refId": "C" +======= + "expr": "sum(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", + "refId": "A" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Flush sink duration percentile", +======= + "title": "Table sink output events", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -3599,9 +5636,9 @@ }, "yaxes": [ { - "format": "s", + "format": "none", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -3704,6 +5741,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Distribution of MySQL worker loads", "fill": 1, "fillGradient": 0, @@ -3718,6 +5756,22 @@ "legend": { "alignAsTable": true, "avg": true, +======= + "description": "The total number of events that buffer sink outputs to backend sink per second", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 222, + "legend": { + "alignAsTable": false, + "avg": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "current": true, "max": true, "min": false, @@ -3735,6 +5789,10 @@ "options": { "dataLinks": [] }, +<<<<<<< HEAD +======= + "paceLength": 10, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "percentage": false, "pointradius": 2, "points": false, @@ -3745,12 +5803,20 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture,bucket)", +======= + "expr": "sum(rate(ticdc_sink_buffer_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "hide": true, "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{capture}}-{{bucket}}", +======= + "legendFormat": "{{changefeed}}-{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { @@ -3802,7 +5868,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "MySQL sink worker load", +======= + "title": "Buffer sink output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -3818,7 +5888,11 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "short", +======= + "format": "none", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -3831,7 +5905,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -3840,6 +5914,7 @@ } }, { +<<<<<<< HEAD "cards": { "cardPadding": 0, "cardRound": 0 @@ -3868,10 +5943,31 @@ "id": 165, "legend": { "alignAsTable": true, +======= + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The total number of events that buffer sink emits", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 226, + "legend": { + "alignAsTable": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "avg": false, "current": true, "max": true, "min": false, +<<<<<<< HEAD "rightSide": true, "show": true, "sort": "current", @@ -3911,11 +6007,83 @@ "max": null, "min": null, "show": true, - "splitFactor": null + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null +======= + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(ticdc_sink_buffer_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Buffer sink output events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "aliasColors": {}, @@ -3923,17 +6091,29 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Percentiles of detecting and waiting conflict duration of MySQL sink", +======= + "description": "The number of rows(events) that are processed by sink per second", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, +<<<<<<< HEAD "x": 12, "y": 61 }, "hiddenSeries": false, "id": 83, +======= + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 227, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, @@ -3942,6 +6122,8 @@ "min": false, "rightSide": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -3962,11 +6144,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-p95", +======= + "expr": "sum (rate(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { @@ -3990,7 +6180,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "MySQL sink conflict detect duration percentile", +======= + "title": "Sink output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -4033,20 +6227,34 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD +======= + "description": "The number of rows(events) that are processed by sink", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "gridPos": { "h": 8, "w": 12, +<<<<<<< HEAD "x": 0, "y": 68 }, "id": 149, +======= + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 225, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": false }, @@ -4054,6 +6262,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", +<<<<<<< HEAD +======= + "options": { + "dataLinks": [] + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "percentage": false, "pointradius": 2, "points": false, @@ -4064,10 +6278,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture)", +======= + "expr": "sum(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", + "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{ capture }}", +======= + "legendFormat": "{{changefeed}}-{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -4075,10 +6298,14 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Processor Memory Consumption Per Capture", +======= + "title": "Sink output events", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -4118,17 +6345,29 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Percentiles of executing DDL duration of MySQL sink", +======= + "description": "The number of rows that sink flushes to downstream per second", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, +<<<<<<< HEAD "x": 12, "y": 68 }, "hiddenSeries": false, "id": 166, +======= + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 108, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, @@ -4137,6 +6376,8 @@ "min": false, "rightSide": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -4157,11 +6398,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", +======= + "expr": "sum(rate(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{instance}}-p95", +======= + "legendFormat": "{{changefeed}}-{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { @@ -4185,7 +6434,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "MySQL sink ddl execution duration percentile", +======= + "title": "Sink flush rows/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -4209,7 +6462,7 @@ "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -4228,20 +6481,33 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD +======= + "description": "The number of rows(events) that are flushed by sink", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, +<<<<<<< HEAD "y": 75 }, "id": 164, +======= + "y": 28 + }, + "hiddenSeries": false, + "id": 240, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": false }, @@ -4249,6 +6515,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", +<<<<<<< HEAD +======= + "options": { + "dataLinks": [] + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "percentage": false, "pointradius": 2, "points": false, @@ -4259,10 +6531,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture, changefeed)", +======= + "expr": "sum(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed, capture)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", + "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{ capture }}-{{ changefeed }}", +======= + "legendFormat": "{{changefeed}}-{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -4270,10 +6551,14 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Processor Memory Consumption Per Changefeed", +======= + "title": "Sink flush rows", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -4306,7 +6591,21 @@ "align": false, "alignLevel": null } - }, + } + ], + "title": "Dataflow", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 13, + "panels": [ { "columns": [], "datasource": "${DS_TEST-CLUSTER}", @@ -4480,24 +6779,184 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of events that puller outputs to sorter \n per second", +======= + "description": "The number of established Eventfeed RPC between TiCDC and TiKV", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 4, - "w": 12, + "h": 7, + "w": 8, "x": 0, +<<<<<<< HEAD "y": 3 }, "hiddenSeries": false, "id": 218, +======= + "y": 5 + }, + "hiddenSeries": false, + "id": 15, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, - "max": true, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*-rpc/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { +<<<<<<< HEAD + "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}", +======= + "expr": "sum(ticdc_kvclient_event_feed_count{tidb_cluster=\"$tidb_cluster\"}) by (instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "refId": "A" + }, + { + "expr": "sum(grpc_client_started_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance) - sum(grpc_client_handled_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-rpc", + "refId": "B" + }, + { + "expr": "sum(grpc_client_started_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}-rpc-started", + "refId": "C" + }, + { + "expr": "sum(grpc_client_handled_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}-rpc-handled", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, +<<<<<<< HEAD + "title": "Puller output events/s", +======= + "title": "Eventfeed count", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD + "description": "The total number of events that puller outputs", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 3 + }, + "hiddenSeries": false, + "id": 229, +======= + "decimals": null, + "description": "Percentiles of Eventfeed message size", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 5 + }, + "hiddenSeries": false, + "id": 17, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -4520,18 +6979,36 @@ "steppedLine": false, "targets": [ { - "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", +<<<<<<< HEAD + "expr": "sum(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance, type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", +======= + "expr": "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p999", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p95", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Puller output events/s", +<<<<<<< HEAD + "title": "Puller output events", +======= + "title": "Event size percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4547,7 +7024,7 @@ }, "yaxes": [ { - "format": "none", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -4560,7 +7037,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -4570,28 +7047,44 @@ }, { "aliasColors": {}, - "bars": false, + "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The total number of events that puller outputs", +<<<<<<< HEAD + "description": "The number of events that sorter outputs to puller \n per second", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 12, - "y": 3 + "x": 0, + "y": 7 }, "hiddenSeries": false, - "id": 229, + "id": 228, +======= + "description": "The number of errors that interrupt Eventfeed RPC", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 5 + }, + "hiddenSeries": false, + "id": 28, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -4614,18 +7107,50 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance, type)", +<<<<<<< HEAD + "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed,type)", +======= + "expr": "sum(increase(ticdc_kvclient_event_feed_error_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (type)", "format": "time_series", + "hide": false, + "interval": "1m", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "{{type}}", + "refId": "A" + }, + { + "expr": "-sum(increase(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))", + "format": "time_series", + "hide": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "transfer-leader", + "refId": "B" + }, + { + "expr": "-sum(increase(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, +<<<<<<< HEAD + "legendFormat": "{{capture}}-{{type}}", "refId": "A" +======= + "legendFormat": "move-region", + "refId": "C" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Puller output events", +<<<<<<< HEAD + "title": "Sorter output events/s", +======= + "title": "Eventfeed error/m", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4641,7 +7166,7 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -4668,21 +7193,37 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of events that sorter outputs to puller \n per second", +<<<<<<< HEAD + "description": "The total number of events that sorter outputs", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 0, + "x": 12, "y": 7 }, "hiddenSeries": false, - "id": 228, + "id": 220, +======= + "description": "The number of KV client received events from TiKV per seconds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 29, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, "rightSide": false, @@ -4708,10 +7249,17 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed,type)", +<<<<<<< HEAD + "expr": "sum(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (capture, changefeed, type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{capture}}-{{type}}", +======= + "expr": "sum(rate(ticdc_kvclient_pull_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -4719,7 +7267,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Sorter output events/s", +<<<<<<< HEAD + "title": "Sorter output events", +======= + "title": "KV client receive events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4735,7 +7287,7 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -4748,7 +7300,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -4762,19 +7314,33 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The total number of events that sorter outputs", +<<<<<<< HEAD + "description": "The number of events that mounter outputs to sink per second", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 12, - "y": 7 + "x": 0, + "y": 11 }, "hiddenSeries": false, - "id": 220, + "id": 219, +======= + "description": "The number of events that puller outputs to sorter \n per second", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 5, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, "max": true, @@ -4802,10 +7368,17 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (capture, changefeed, type)", +<<<<<<< HEAD + "expr": "sum(rate(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{type}}", + "legendFormat": "{{capture}}", +======= + "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -4813,7 +7386,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Sorter output events", +<<<<<<< HEAD + "title": "Mounter output events/s", +======= + "title": "Puller output events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4856,21 +7433,36 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of events that mounter outputs to sink per second", +<<<<<<< HEAD + "description": "The total number of events that mounter outputs", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 0, + "x": 12, "y": 11 }, "hiddenSeries": false, - "id": 219, + "id": 224, +======= + "description": "The number of events that are buffered in Processor's output channel and Mounter input channel", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 107, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, "rightSide": false, @@ -4896,18 +7488,33 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", +<<<<<<< HEAD + "expr": "sum(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance)", +======= + "expr": "sum(ticdc_mounter_input_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}", + "legendFormat": "{{capture}}-mounter input chan", "refId": "A" + }, + { + "expr": "-sum(ticdc_sink_buffer_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-sink buffer chan", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Mounter output events/s", +<<<<<<< HEAD + "title": "Mounter output events", +======= + "title": "Sink/Mounter buffer size", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4923,7 +7530,7 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -4936,7 +7543,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -4944,30 +7551,119 @@ "alignLevel": null } }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "Duration of sorting unsorted events", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 19 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Entry sorter sort duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The total number of events that mounter outputs", +<<<<<<< HEAD + "description": "The number of events that table sorter outputs to buffer sink per second", +======= + "description": "Percentiles of sorting events duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 4, + "h": 7, "w": 12, +<<<<<<< HEAD + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 223, +======= "x": 12, - "y": 11 + "y": 19 }, "hiddenSeries": false, - "id": 224, + "id": 53, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -4990,18 +7686,37 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance)", +<<<<<<< HEAD + "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", +======= + "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-p999", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p95", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Mounter output events", +<<<<<<< HEAD + "title": "Table sink output events/s", +======= + "title": "Entry sorter sort duration percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -5017,9 +7732,9 @@ }, "yaxes": [ { - "format": "none", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -5039,98 +7754,80 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of events that table sorter outputs to buffer sink per second", - "fill": 1, - "fillGradient": 0, + "description": "Duration of merging sorted events", "gridPos": { - "h": 4, + "h": 7, "w": 12, "x": 0, - "y": 15 + "y": 26 }, - "hiddenSeries": false, - "id": 223, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 105, "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", + "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Table sink output events/s", + "title": "Entry sorter merge duration", "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "show": true, + "showHistogram": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null }, { "aliasColors": {}, @@ -5138,24 +7835,36 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The total number of events that table sinks emit", +======= + "description": "Percentiles of merging sorted events duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { - "h": 4, + "h": 7, "w": 12, "x": 12, +<<<<<<< HEAD "y": 15 }, "hiddenSeries": false, "id": 221, +======= + "y": 26 + }, + "hiddenSeries": false, + "id": 106, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -5178,18 +7887,37 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", +======= + "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-p999", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p95", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Table sink output events", +======= + "title": "Entry sorter merge duration percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -5205,9 +7933,9 @@ }, "yaxes": [ { - "format": "none", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -5227,36 +7955,57 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The total number of events that buffer sink outputs to backend sink per second", "fill": 1, "fillGradient": 0, +======= + "description": "Duration of unmarshal events from kv to SQL row", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "gridPos": { - "h": 4, + "h": 7, "w": 12, "x": 0, +<<<<<<< HEAD "y": 19 }, "hiddenSeries": false, "id": 222, +======= + "y": 33 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 101, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true }, - "lines": true, - "linewidth": 1, "links": [], +<<<<<<< HEAD "nullPointMode": "null", "options": { "dataLinks": [] @@ -5288,15 +8037,41 @@ "shared": true, "sort": 0, "value_type": "individual" +======= + "reverseYBuckets": false, + "targets": [ + { + "expr": "max(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Mounter unmarshal duration", + "tooltip": { + "show": true, + "showHistogram": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, +<<<<<<< HEAD "yaxes": [ { "format": "none", @@ -5319,6 +8094,11 @@ "align": false, "alignLevel": null } +======= + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "aliasColors": {}, @@ -5326,24 +8106,36 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The total number of events that buffer sink emits", "fill": 1, +======= + "description": "Percentiles of unmarshal events from kv to SQL row duration", + "fill": 0, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fillGradient": 0, "gridPos": { - "h": 4, + "h": 7, "w": 12, "x": 12, +<<<<<<< HEAD "y": 19 }, "hiddenSeries": false, "id": 226, +======= + "y": 33 + }, + "hiddenSeries": false, + "id": 55, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -5366,18 +8158,39 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_sink_buffer_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", +======= + "expr": "histogram_quantile(0.99, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p99", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" + }, + { + "expr": "histogram_quantile(0.999, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", + "format": "time_series", + "hide": true, + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p999", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Buffer sink output events", +======= + "title": "Mounter unmarshal duration percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -5393,7 +8206,7 @@ }, "yaxes": [ { - "format": "none", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -5401,7 +8214,11 @@ "show": true }, { +<<<<<<< HEAD "format": "short", +======= + "format": "none", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -5420,6 +8237,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of rows(events) that are processed by sink per second", "fill": 1, "fillGradient": 0, @@ -5431,10 +8249,28 @@ }, "hiddenSeries": false, "id": 227, +======= + "description": "The number of KV client dispatched event per second", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 31, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": false, "avg": false, "current": true, +<<<<<<< HEAD +======= + "hideEmpty": true, + "hideZero": true, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": true, "min": false, "rightSide": false, @@ -5455,25 +8291,54 @@ "pointradius": 2, "points": false, "renderer": "flot", +<<<<<<< HEAD "seriesOverrides": [], +======= + "seriesOverrides": [ + { + "alias": "/.*batch-resolved/", + "yaxis": 2 + } + ], +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum (rate(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture)", +======= + "expr": "sum(rate(ticdc_kvclient_send_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture, changefeed, type)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{capture}}", "refId": "A" +======= + "legendFormat": "{{capture}}-{{changefeed}}-{{type}}", + "refId": "A" + }, + { + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture, changefeed, table)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{changefeed}}-batch-resolved", + "refId": "B" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Sink output events/s", +======= + "title": "KV client dispatch events/s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -5510,12 +8375,89 @@ "alignLevel": null } }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The size of batch resolved ts message from TiKV", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 40 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 97, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "KV client batch resolved size", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "none", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of rows(events) that are processed by sink", "fill": 1, "fillGradient": 0, @@ -5527,10 +8469,28 @@ }, "hiddenSeries": false, "id": 225, +======= + "description": "The number of regions that are being scanned", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 177, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": false, "avg": false, "current": true, +<<<<<<< HEAD +======= + "hideEmpty": true, + "hideZero": true, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": true, "min": false, "rightSide": false, @@ -5557,11 +8517,19 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (capture, type)", +======= + "expr": "sum(ticdc_kvclient_region_token{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{capture}}", +======= + "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -5569,7 +8537,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Sink output events", +======= + "title": "KV client scanning regions", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -5585,7 +8557,11 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "none", +======= + "format": "short", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -5612,6 +8588,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of rows that sink flushes to downstream per second", "fill": 1, "fillGradient": 0, @@ -5623,6 +8600,19 @@ }, "hiddenSeries": false, "id": 108, +======= + "description": "Active stream count of each gRPC connection", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 188, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": false, "avg": false, @@ -5653,10 +8643,17 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(rate(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{capture}}", +======= + "expr": "sum(ticdc_kvclient_grpc_stream_count{tidb_cluster=\"$tidb_cluster\"}) by (store)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{store}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -5664,7 +8661,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Sink flush rows/s", +======= + "title": "KV client gRPC stream count", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -5707,6 +8708,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of rows(events) that are flushed by sink", "fill": 1, "fillGradient": 0, @@ -5718,13 +8720,32 @@ }, "hiddenSeries": false, "id": 240, +======= + "description": "The number of regions that have not connected to TiKV", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 251, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": false, "avg": false, "current": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sort": "current", "sortDesc": true, @@ -5736,7 +8757,7 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, "pointradius": 2, @@ -5748,11 +8769,20 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (capture, type)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{capture}}", +======= + "exemplar": true, + "expr": "sum(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -5760,7 +8790,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Sink flush rows", +======= + "title": "KV client cached regions", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 2, @@ -5817,23 +8851,48 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of established Eventfeed RPC between TiCDC and TiKV", +======= + "description": "Estimate the remaining time for a changefeed initialization (on a specific capture)", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 8, "x": 0, "y": 32 }, "hiddenSeries": false, "id": 15, +======= + "w": 12, + "x": 12, + "y": 54 + }, + "hiddenSeries": false, + "id": 252, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, "current": true, +<<<<<<< HEAD "hideEmpty": false, "max": false, +======= + "hideEmpty": true, + "hideZero": true, + "max": true, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "min": false, "rightSide": true, "show": true, @@ -5845,7 +8904,7 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "paceLength": 10, "percentage": false, @@ -5863,10 +8922,20 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_kvclient_event_feed_count{tidb_cluster=\"$tidb_cluster\"}) by (instance)", +======= + "exemplar": true, + "expr": "abs(sum(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"} / deriv(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture, changefeed, store))", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", + "interval": "", "intervalFactor": 1, +<<<<<<< HEAD "legendFormat": "{{instance}}", +======= + "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { @@ -5897,7 +8966,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Eventfeed count", +======= + "title": "Estimate remaining time for initialization", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -5913,9 +8986,15 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "short", "label": null, "logBase": 1, +======= + "format": "s", + "label": "", + "logBase": 2, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": null, "min": null, "show": true @@ -5933,18 +9012,35 @@ "align": false, "alignLevel": null } - }, + } + ], + "title": "Events", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 130, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "decimals": null, "description": "Percentiles of Eventfeed message size", +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, - "fillGradient": 0, "gridPos": { +<<<<<<< HEAD "h": 7, "w": 8, "x": 8, @@ -5952,25 +9048,34 @@ }, "hiddenSeries": false, "id": 17, +======= + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 131, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, "avg": false, +<<<<<<< HEAD "current": true, "max": false, "min": false, "rightSide": true, +======= + "current": false, + "max": false, + "min": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -5981,6 +9086,7 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", "format": "time_series", "intervalFactor": 1, @@ -5993,13 +9099,24 @@ "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}-p95", "refId": "B" +======= + "expr": "sum(rate(ticdc_sorter_consume_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{changefeed}}", + "refId": "A" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Event size percentile", +======= + "title": "Unified Sorter intake rate", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -6037,6 +9154,7 @@ } }, { +<<<<<<< HEAD "aliasColors": {}, "bars": true, "dashLength": 10, @@ -6152,15 +9270,20 @@ } }, { +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of KV client received events from TiKV per seconds", +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, - "fillGradient": 0, "gridPos": { +<<<<<<< HEAD "h": 7, "w": 8, "x": 0, @@ -6168,27 +9291,36 @@ }, "hiddenSeries": false, "id": 29, +======= + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 132, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, "avg": false, +<<<<<<< HEAD "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": false, +======= + "current": false, + "max": false, + "min": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -6199,10 +9331,17 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(rate(ticdc_kvclient_pull_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", +======= + "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{changefeed}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -6210,7 +9349,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "KV client receive events/s", +======= + "title": "Unified Sorter event output rate", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -6253,10 +9396,13 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of events that puller outputs to sorter \n per second", +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, - "fillGradient": 0, "gridPos": { +<<<<<<< HEAD "h": 7, "w": 8, "x": 8, @@ -6271,18 +9417,27 @@ "max": true, "min": false, "rightSide": false, +======= + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 133, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -6293,10 +9448,17 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", +======= + "expr": "sum(ticdc_sorter_on_disk_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -6304,7 +9466,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "Puller output events/s", +======= + "title": "Unified Sorter on disk data size", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -6320,12 +9486,146 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "none", +======= + "format": "bytes", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD + "description": "The number of events that are buffered in Processor's output channel and Mounter input channel", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "hiddenSeries": false, + "id": 107, +======= + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 134, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legend": { + "avg": false, +<<<<<<< HEAD + "current": true, + "hideEmpty": false, + "max": true, + "min": false, + "rightSide": false, +======= + "current": false, + "max": false, + "min": false, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { +<<<<<<< HEAD + "expr": "sum(ticdc_mounter_input_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-mounter input chan", + "refId": "A" + }, + { + "expr": "-sum(ticdc_sink_buffer_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}-sink buffer chan", + "refId": "B" +======= + "expr": "sum(ticdc_sorter_in_memory_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}", + "refId": "A" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, +<<<<<<< HEAD + "title": "Sink/Mounter buffer size", +======= + "title": "Unified Sorter in-memory data size", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { +<<<<<<< HEAD + "format": "short", +======= + "format": "bytes", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true }, { "format": "short", @@ -6341,23 +9641,117 @@ "alignLevel": null } }, + { + "cards": { +<<<<<<< HEAD + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "Duration of sorting unsorted events", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 46 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Entry sorter sort duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of events that are buffered in Processor's output channel and Mounter input channel", - "fill": 1, - "fillGradient": 0, + "description": "Percentiles of sorting events duration", + "fill": 1, + "fillGradient": 0, +======= + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 39 + "h": 8, + "w": 12, +<<<<<<< HEAD + "x": 12, + "y": 46 }, "hiddenSeries": false, - "id": 107, + "id": 53, "legend": { "alignAsTable": true, "avg": false, @@ -6365,14 +9759,24 @@ "hideEmpty": false, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true +======= + "x": 0, + "y": 21 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 135, + "legend": { + "show": false +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "lines": true, - "linewidth": 1, "links": [], +<<<<<<< HEAD "nullPointMode": "null", "options": { "dataLinks": [] @@ -6388,43 +9792,63 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_mounter_input_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-mounter input chan", + "legendFormat": "{{capture}}-p999", "refId": "A" }, { - "expr": "-sum(ticdc_sink_buffer_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "{{capture}}-sink buffer chan", + "legendFormat": "{{capture}}-p95", "refId": "B" +======= + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(ticdc_sorter_flush_count_histogram_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "A" +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Sink/Mounter buffer size", +<<<<<<< HEAD + "title": "Entry sorter sort duration percentile", +======= + "title": "Unified Sorter flush sizes", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "show": true, + "showHistogram": false }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "none", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, +<<<<<<< HEAD "yaxes": [ { - "format": "short", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -6442,64 +9866,78 @@ "align": false, "alignLevel": null } +======= + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "cards": { - "cardPadding": 0, - "cardRound": 0 + "cardPadding": null, + "cardRound": null }, "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateBlues", "exponent": 0.5, - "min": 0, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "Duration of sorting unsorted events", +<<<<<<< HEAD + "description": "Duration of merging sorted events", +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "gridPos": { - "h": 7, + "h": 8, "w": 12, +<<<<<<< HEAD "x": 0, - "y": 46 + "y": 53 +======= + "x": 12, + "y": 21 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 99, +<<<<<<< HEAD + "id": 105, +======= + "id": 136, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true + "show": false }, "links": [], "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", +<<<<<<< HEAD + "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", +======= + "expr": "sum(rate(ticdc_sorter_merge_count_histogram_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (le)", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "heatmap", - "instant": false, - "intervalFactor": 2, + "intervalFactor": 1, "legendFormat": "{{le}}", "refId": "A" } ], - "title": "Entry sorter sort duration", +<<<<<<< HEAD + "title": "Entry sorter merge duration", +======= + "timeFrom": null, + "timeShift": null, + "title": "Unified Sorter merge size", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "show": true, - "showHistogram": true + "showHistogram": false }, - "tooltipDecimals": 1, "type": "heatmap", "xAxis": { "show": true @@ -6507,103 +9945,290 @@ "xBucketNumber": null, "xBucketSize": null, "yAxis": { +<<<<<<< HEAD "decimals": 1, "format": "s", +======= + "decimals": null, + "format": "none", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "logBase": 1, "max": null, "min": null, "show": true, "splitFactor": null }, - "yBucketBound": "upper", + "yBucketBound": "auto", "yBucketNumber": null, "yBucketSize": null + } + ], + "title": "Unified Sorter", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 266, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD + "description": "Percentiles of merging sorted events duration", +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, +<<<<<<< HEAD + "x": 12, + "y": 53 + }, + "hiddenSeries": false, + "id": 106, +======= + "x": 0, + "y": 16 + }, + "id": 289, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { +<<<<<<< HEAD + "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", +======= + "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{tidb_cluster=\"$tidb_cluster\", job=\"pd\"}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "format": "time_series", + "hide": false, + "intervalFactor": 1, +<<<<<<< HEAD + "legendFormat": "{{capture}}-p999", +======= + "legendFormat": "{{instance}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p95", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, +<<<<<<< HEAD + "title": "Entry sorter merge duration percentile", +======= + "title": "Etcd MVCC DB total size", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { +<<<<<<< HEAD + "format": "s", +======= + "format": "decbytes", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "Percentiles of sorting events duration", +<<<<<<< HEAD + "description": "Duration of unmarshal events from kv to SQL row", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 60 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 101, +======= "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 46 + "y": 16 }, "hiddenSeries": false, - "id": 53, + "id": 114, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, +<<<<<<< HEAD +======= + "hideEmpty": true, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": true, "min": false, "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "reverseYBuckets": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", +<<<<<<< HEAD + "expr": "max(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", +======= + "expr": "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-p999", + "legendFormat": "p999-{{instance}}", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "{{capture}}-p95", + "legendFormat": "p99-{{instance}}", "refId": "B" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p95-{{instance}}", + "refId": "C" } ], +<<<<<<< HEAD + "title": "Mounter unmarshal duration", +======= "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Entry sorter sort duration percentile", + "title": "Etcd health check duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "show": true, + "showHistogram": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, +<<<<<<< HEAD + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null +======= "yaxes": [ { "format": "s", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -6621,6 +10246,7 @@ "align": false, "alignLevel": null } +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "cards": { @@ -6628,26 +10254,28 @@ "cardRound": 0 }, "color": { - "cardColor": "#FF9830", - "colorScale": "linear", + "cardColor": "#b4ff00", + "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, - "min": 0, + "max": null, + "min": 1, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "Duration of merging sorted events", +<<<<<<< HEAD + "description": "Percentiles of unmarshal events from kv to SQL row duration", + "fill": 0, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 53 + "x": 12, + "y": 60 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 105, + "hiddenSeries": false, + "id": 55, "legend": { "alignAsTable": true, "avg": false, @@ -6661,22 +10289,78 @@ "total": false, "values": true }, + "lines": true, + "linewidth": 1, "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] +======= + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 267, + "legend": { + "show": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + }, + "pluginVersion": "6.1.6", "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", +<<<<<<< HEAD + "expr": "histogram_quantile(0.99, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p99", +======= + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", "format": "heatmap", "instant": false, + "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", + "queryType": "randomWalk", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" + }, + { + "expr": "histogram_quantile(0.999, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", + "format": "time_series", + "hide": true, + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{capture}}-p999", + "refId": "B" } ], - "title": "Entry sorter merge duration", + "timeFrom": null, + "timeShift": null, +<<<<<<< HEAD + "title": "Mounter unmarshal duration percentile", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" +======= + "title": "EtcdWorker exec etcd txn duration", "tooltip": { "show": true, "showHistogram": true +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "tooltipDecimals": 1, "type": "heatmap", @@ -6694,9 +10378,34 @@ "show": true, "splitFactor": null }, +<<<<<<< HEAD + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } +======= "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "aliasColors": {}, @@ -6704,59 +10413,91 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Percentiles of merging sorted events duration", +<<<<<<< HEAD + "description": "The number of KV client dispatched event per second", +======= + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, +<<<<<<< HEAD + "x": 0, + "y": 67 + }, + "hiddenSeries": false, + "id": 31, +======= "x": 12, - "y": 53 + "y": 23 }, "hiddenSeries": false, - "id": 106, + "id": 264, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": false, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "dataLinks": [] }, - "paceLength": 10, "percentage": false, + "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*batch-resolved/", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", +<<<<<<< HEAD + "expr": "sum(rate(ticdc_kvclient_send_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture, changefeed, type)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-p999", + "legendFormat": "{{capture}}-{{changefeed}}-{{type}}", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture, changefeed, table)", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "{{capture}}-p95", + "legendFormat": "{{capture}}-{{changefeed}}-batch-resolved", +======= + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "interval": "", + "legendFormat": "{{capture}}-95", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "hide": false, + "interval": "", + "legendFormat": "{{capture}}-99", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "B" } ], @@ -6764,7 +10505,11 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Entry sorter merge duration percentile", +<<<<<<< HEAD + "title": "KV client dispatch events/s", +======= + "title": "EtcdWorker tick reactor duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -6780,15 +10525,19 @@ }, "yaxes": [ { +<<<<<<< HEAD + "format": "short", +======= "format": "s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -6801,58 +10550,60 @@ "alignLevel": null } }, +<<<<<<< HEAD +======= { "cards": { "cardPadding": 0, "cardRound": 0 }, "color": { - "cardColor": "#FF9830", - "colorScale": "linear", + "cardColor": "#b4ff00", + "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, - "min": 0, + "max": null, + "min": 1, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "Duration of unmarshal events from kv to SQL row", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 60 + "y": 30 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 101, + "id": 256, "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true + "show": true }, - "links": [], + "pluginVersion": "6.1.6", "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", "format": "heatmap", "instant": false, + "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "Mounter unmarshal duration", + "timeFrom": null, + "timeShift": null, + "title": "EtcdWorker exec etcd txn duration", "tooltip": { "show": true, "showHistogram": true @@ -6877,23 +10628,52 @@ "yBucketNumber": null, "yBucketSize": null }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "Percentiles of unmarshal events from kv to SQL row duration", - "fill": 0, +<<<<<<< HEAD + "description": "The size of batch resolved ts message from TiKV", +======= + "description": "", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, "fillGradient": 0, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 60 +<<<<<<< HEAD + "y": 67 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 97, +======= + "y": 30 }, "hiddenSeries": false, - "id": 55, + "id": 258, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, @@ -6907,172 +10687,94 @@ "total": false, "values": true }, - "lines": true, - "linewidth": 1, +<<<<<<< HEAD "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "reverseYBuckets": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{capture}}-p99", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", - "format": "time_series", - "hide": true, + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", + "format": "heatmap", "instant": false, - "intervalFactor": 1, - "legendFormat": "{{capture}}-p999", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Mounter unmarshal duration percentile", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of KV client dispatched event per second", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 67 - }, - "hiddenSeries": false, - "id": 31, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, + "intervalFactor": 2, + "legendFormat": "{{le}}", +======= "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, - "paceLength": 10, "percentage": false, + "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*batch-resolved/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_kvclient_send_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture, changefeed, type)", + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}-{{type}}", + "legendFormat": "{{capture}}-p95", + "queryType": "randomWalk", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" }, { - "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture, changefeed, table)", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", + "hide": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}-batch-resolved", + "legendFormat": "{{capture}}-p99", "refId": "B" } ], +<<<<<<< HEAD + "title": "KV client batch resolved size", +======= "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV client dispatch events/s", + "title": "EtcdWorker exec etcd txn duration percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "show": true, + "showHistogram": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "none", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, +<<<<<<< HEAD + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null +======= "yaxes": [ { - "format": "short", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -7090,64 +10792,62 @@ "align": false, "alignLevel": null } +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "cards": { - "cardPadding": 0, - "cardRound": 0 + "cardPadding": null, + "cardRound": null }, "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolatePurples", "exponent": 0.5, "min": 0, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "The size of batch resolved ts message from TiKV", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 67 + "x": 0, + "y": 37 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 97, + "id": 254, "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true + "show": true }, - "links": [], + "pluginVersion": "6.1.6", "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", "format": "heatmap", "instant": false, - "intervalFactor": 2, + "interval": "", + "intervalFactor": 1, "legendFormat": "{{le}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "KV client batch resolved size", + "title": "EtcdWorker txn size", "tooltip": { "show": true, "showHistogram": true }, - "tooltipDecimals": 1, + "tooltipDecimals": null, "type": "heatmap", "xAxis": { "show": true @@ -7155,15 +10855,15 @@ "xBucketNumber": null, "xBucketSize": null, "yAxis": { - "decimals": 1, - "format": "none", + "decimals": null, + "format": "decbytes", "logBase": 1, "max": null, "min": null, "show": true, "splitFactor": null }, - "yBucketBound": "upper", + "yBucketBound": "auto", "yBucketNumber": null, "yBucketSize": null }, @@ -7173,17 +10873,33 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of regions that are being scanned", +======= + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, +<<<<<<< HEAD "x": 0, "y": 74 }, "hiddenSeries": false, "id": 177, +======= + "x": 12, + "y": 37 + }, + "hiddenSeries": false, + "id": 260, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, @@ -7199,13 +10915,18 @@ }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { +<<<<<<< HEAD "dataLinks": [] }, "paceLength": 10, +======= + "alertThreshold": true + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "percentage": false, + "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", @@ -7215,18 +10936,38 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_kvclient_region_token{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", +======= + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "interval": "", + "legendFormat": "{{capture}}-p95", + "queryType": "randomWalk", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "hide": false, + "interval": "", + "legendFormat": "{{capture}}-p99", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "KV client scanning regions", +======= + "title": "EtcdWorker txn size percentile", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -7242,7 +10983,7 @@ }, "yaxes": [ { - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -7269,6 +11010,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "Active stream count of each gRPC connection", "fill": 1, "fillGradient": 0, @@ -7280,6 +11022,20 @@ }, "hiddenSeries": false, "id": 188, +======= + "description": "The time consumed of writing WAL into the persistent storage in .99", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 291, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, @@ -7290,6 +11046,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -7297,12 +11054,15 @@ "linewidth": 1, "links": [], "nullPointMode": "null", +<<<<<<< HEAD "options": { "dataLinks": [] }, +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -7311,22 +11071,37 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "expr": "sum(ticdc_kvclient_grpc_stream_count{tidb_cluster=\"$tidb_cluster\"}) by (store)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{store}}", "refId": "A" +======= + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 4 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "KV client gRPC stream count", +======= + "title": "Etcd 99% WAL fsync duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -7338,7 +11113,11 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "short", +======= + "format": "s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -7365,6 +11144,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", +<<<<<<< HEAD "description": "The number of regions that have not connected to TiKV", "fieldConfig": { "defaults": { @@ -7382,16 +11162,34 @@ }, "hiddenSeries": false, "id": 251, +======= + "description": "The time consumed of handling etcd transactions in .99", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 290, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": true, "avg": false, "current": true, +<<<<<<< HEAD "hideEmpty": true, "hideZero": true, +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -7399,12 +11197,15 @@ "linewidth": 1, "links": [], "nullPointMode": "null", +<<<<<<< HEAD "options": { "alertThreshold": true }, +======= +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -7413,6 +11214,7 @@ "steppedLine": false, "targets": [ { +<<<<<<< HEAD "exemplar": true, "expr": "sum(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", "format": "time_series", @@ -7420,17 +11222,30 @@ "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", "refId": "A" +======= + "expr": "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, result, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{result}}", + "refId": "A", + "step": 4 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, +<<<<<<< HEAD "title": "KV client cached regions", +======= + "title": "Etcd 99% Handle transactions duration", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -7442,7 +11257,11 @@ }, "yaxes": [ { +<<<<<<< HEAD "format": "short", +======= + "format": "s", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -7462,6 +11281,7 @@ "align": false, "alignLevel": null } +<<<<<<< HEAD }, { "aliasColors": {}, @@ -7569,6 +11389,11 @@ } ], "title": "Events", +======= + } + ], + "title": "Etcd", +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "type": "row" }, { @@ -7592,7 +11417,7 @@ "h": 8, "w": 12, "x": 0, - "y": 5 + "y": 7 }, "id": 131, "legend": { @@ -7677,7 +11502,7 @@ "h": 8, "w": 12, "x": 12, - "y": 5 + "y": 7 }, "id": 132, "legend": { @@ -7762,7 +11587,11 @@ "h": 8, "w": 12, "x": 0, +<<<<<<< HEAD "y": 13 +======= + "y": 14 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 133, "legend": { @@ -7847,7 +11676,11 @@ "h": 8, "w": 12, "x": 12, +<<<<<<< HEAD "y": 13 +======= + "y": 14 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 134, "legend": { @@ -8078,7 +11911,11 @@ "h": 7, "w": 12, "x": 0, +<<<<<<< HEAD "y": 5 +======= + "y": 21 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 60, "legend": { @@ -8202,9 +12039,15 @@ "grid": {}, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 12, "x": 12, "y": 5 +======= + "w": 5, + "x": 7, + "y": 21 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 74, "legend": { @@ -8311,8 +12154,13 @@ "gridPos": { "h": 7, "w": 12, +<<<<<<< HEAD "x": 0, "y": 12 +======= + "x": 12, + "y": 21 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 147, "legend": { @@ -8396,6 +12244,88 @@ } }, { +<<<<<<< HEAD +======= + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The time consumed to CDC incremental scan", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 68, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "maxPerRow": 3, + "repeat": null, + "repeatDirection": "h", + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(tikv_cdc_scan_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Initial scan duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "aliasColors": {}, "bars": false, "dashLength": 10, @@ -8411,7 +12341,11 @@ "h": 7, "w": 12, "x": 12, +<<<<<<< HEAD "y": 12 +======= + "y": 28 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 194, "legend": { @@ -8573,9 +12507,15 @@ "grid": {}, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 7, "x": 0, "y": 19 +======= + "w": 6, + "x": 18, + "y": 28 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 152, "legend": { @@ -8704,9 +12644,15 @@ "grid": {}, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 5, "x": 7, "y": 19 +======= + "w": 12, + "x": 0, + "y": 35 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 153, "legend": { @@ -8802,7 +12748,11 @@ "h": 7, "w": 12, "x": 12, +<<<<<<< HEAD "y": 19 +======= + "y": 35 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 70, "legend": { @@ -8975,8 +12925,13 @@ "gridPos": { "h": 7, "w": 6, +<<<<<<< HEAD "x": 12, "y": 26 +======= + "x": 18, + "y": 35 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 72, "legend": { @@ -9069,9 +13024,15 @@ "fill": 1, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 6, "x": 18, "y": 26 +======= + "w": 12, + "x": 0, + "y": 42 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 140, "legend": { @@ -9182,8 +13143,13 @@ "gridPos": { "h": 7, "w": 12, +<<<<<<< HEAD "x": 0, "y": 33 +======= + "x": 12, + "y": 42 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 78, "legend": { @@ -9289,9 +13255,15 @@ "grid": {}, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 6, "x": 12, "y": 33 +======= + "w": 12, + "x": 0, + "y": 49 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 76, "legend": { @@ -9391,8 +13363,13 @@ "gridPos": { "h": 7, "w": 6, +<<<<<<< HEAD "x": 18, "y": 33 +======= + "x": 12, + "y": 49 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 139, "legend": { @@ -9488,9 +13465,15 @@ "fill": 4, "gridPos": { "h": 7, +<<<<<<< HEAD "w": 12, "x": 0, "y": 40 +======= + "w": 6, + "x": 18, + "y": 49 +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 143, "legend": { @@ -9645,6 +13628,23 @@ "pointradius": 1, "points": false, "renderer": "flot", +<<<<<<< HEAD +======= + "repeat": null, + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "172.16.5.33:47912", + "value": "172.16.5.33:47912" + }, + "runtime_instance": { + "selected": false, + "text": "172.16.5.37:47912", + "value": "172.16.5.37:47912" + } + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "seriesOverrides": [ { "alias": "/.*len/", @@ -9766,6 +13766,23 @@ "pointradius": 5, "points": false, "renderer": "flot", +<<<<<<< HEAD +======= + "repeat": null, + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "172.16.5.33:47912", + "value": "172.16.5.33:47912" + }, + "runtime_instance": { + "selected": false, + "text": "172.16.5.37:47912", + "value": "172.16.5.37:47912" + } + }, +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -9942,7 +13959,25 @@ "pointradius": 5, "points": false, "renderer": "flot", +<<<<<<< HEAD "seriesOverrides": [], +======= + "scopedVars": { + "instance": { + "selected": false, + "text": "172.16.5.33:47912", + "value": "172.16.5.33:47912" + }, + "runtime_instance": { + "selected": false, + "text": "172.16.5.37:47912", + "value": "172.16.5.37:47912" + } + }, + "seriesOverrides": [ + {} + ], +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "spaceLength": 10, "stack": false, "steppedLine": false, @@ -10077,8 +14112,8 @@ }, "runtime_instance": { "selected": false, - "text": "172.16.5.32:47800", - "value": "172.16.5.32:47800" + "text": "172.16.5.37:47912", + "value": "172.16.5.37:47912" } }, "seriesOverrides": [ @@ -10794,6 +14829,11 @@ "timezone": "browser", "title": "Test-Cluster-TiCDC", "uid": "YiGL8hBZ1", +<<<<<<< HEAD "version": 25 } +======= + "version": 29 +} +>>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) From c6b372f706ee03923221de153e3b763d03f6dfc7 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Wed, 29 Dec 2021 00:07:11 +0800 Subject: [PATCH 2/3] fix build Signed-off-by: Neil Shen --- cdc/owner/changefeed.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/cdc/owner/changefeed.go b/cdc/owner/changefeed.go index 4e5b1d7c59f..eef436680f3 100644 --- a/cdc/owner/changefeed.go +++ b/cdc/owner/changefeed.go @@ -260,18 +260,9 @@ LOOP: // init metrics c.metricsChangefeedCheckpointTsGauge = changefeedCheckpointTsGauge.WithLabelValues(c.id) c.metricsChangefeedCheckpointTsLagGauge = changefeedCheckpointTsLagGauge.WithLabelValues(c.id) -<<<<<<< HEAD -======= c.metricsChangefeedResolvedTsGauge = changefeedResolvedTsGauge.WithLabelValues(c.id) c.metricsChangefeedResolvedTsLagGauge = changefeedResolvedTsLagGauge.WithLabelValues(c.id) - // create scheduler - c.scheduler, err = c.newScheduler(ctx, checkpointTs) - if err != nil { - return errors.Trace(err) - } - ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) c.initialized = true return nil } @@ -293,11 +284,7 @@ func (c *changefeed) releaseResources() { log.Warn("Closing sink failed in Owner", zap.String("changefeedID", c.state.ID), zap.Error(err)) } c.wg.Wait() -<<<<<<< HEAD -======= - c.scheduler.Close(ctx) ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) changefeedCheckpointTsGauge.DeleteLabelValues(c.id) changefeedCheckpointTsLagGauge.DeleteLabelValues(c.id) c.metricsChangefeedCheckpointTsGauge = nil From c3d7dd69377568f178f7e8f1d5467ab4f583385b Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Wed, 29 Dec 2021 00:50:55 +0800 Subject: [PATCH 3/3] resolve conflicts Signed-off-by: Neil Shen --- metrics/grafana/ticdc.json | 6626 +++++++----------------------------- 1 file changed, 1217 insertions(+), 5409 deletions(-) diff --git a/metrics/grafana/ticdc.json b/metrics/grafana/ticdc.json index d992b807725..b298d19ff01 100644 --- a/metrics/grafana/ticdc.json +++ b/metrics/grafana/ticdc.json @@ -125,11 +125,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, -<<<<<<< HEAD - "iteration": 1636101093950, -======= "iteration": 1640447119260, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "links": [], "panels": [ { @@ -924,7 +920,6 @@ }, { "collapsed": true, - "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -934,50 +929,21 @@ "id": 11, "panels": [ { -<<<<<<< HEAD - "cards": { - "cardPadding": 1, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": null, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", -======= "columns": [ { "text": "Current", "value": "current" } ], ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "datasource": "${DS_TEST-CLUSTER}", "description": "The number of captured table of TiCDC nodes ", "fontSize": "100%", "gridPos": { -<<<<<<< HEAD - "h": 8, - "w": 12, -======= "h": 5, "w": 7, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "x": 0, "y": 3 }, -<<<<<<< HEAD - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 262, - "legend": { - "show": true -======= "id": 4, "links": [], "pageSize": null, @@ -986,7 +952,6 @@ "sort": { "col": 0, "desc": true ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "styles": [ { @@ -1012,42 +977,6 @@ ], "targets": [ { -<<<<<<< HEAD - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, capture)", - "format": "heatmap", - "interval": "1", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "EtcdWorker tick reactor duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "tooltipDecimals": null, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null -======= "expr": "sum(ticdc_processor_num_of_tables{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", "format": "time_series", "interval": "", @@ -1061,7 +990,6 @@ "title": "Changefeed table count", "transform": "timeseries_aggregations", "type": "table" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "columns": [ @@ -1074,28 +1002,10 @@ "description": "Internal resolved ts of TiCDC nodes", "fontSize": "100%", "gridPos": { -<<<<<<< HEAD - "h": 8, - "w": 12, - "x": 12, - "y": 2 - }, - "hiddenSeries": false, - "id": 264, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false -======= "h": 10, "w": 7, "x": 7, "y": 3 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 90, "links": [], @@ -1190,24 +1100,6 @@ }, "styles": [ { -<<<<<<< HEAD - "$$hashKey": "object:2612", - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:2613", - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true -======= "alias": "table", "colorMode": null, "colors": [ @@ -1237,7 +1129,6 @@ "thresholds": [], "type": "date", "unit": "short" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "targets": [ @@ -1276,17 +1167,10 @@ "description": "The number of replicated tables maintained in owner", "fontSize": "100%", "gridPos": { -<<<<<<< HEAD - "h": 8, - "w": 12, - "x": 0, - "y": 10 -======= "h": 5, "w": 7, "x": 0, "y": 8 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 138, "links": [], @@ -1355,17 +1239,10 @@ "description": "The checkpoint ts of changefeeds.", "fill": 0, "gridPos": { -<<<<<<< HEAD - "h": 8, - "w": 12, - "x": 12, - "y": 10 -======= "h": 7, "w": 9, "x": 0, "y": 13 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 86, "legend": { @@ -1399,22 +1276,10 @@ "steppedLine": false, "targets": [ { - "expr": "max(pd_cluster_tso{tidb_cluster=\"$tidb_cluster\"})", + "expr": "max(pd_cluster_tso{tidb_cluster=\"$tidb_cluster\"}) * 1000", "format": "time_series", "hide": false, "interval": "", -<<<<<<< HEAD - "legendFormat": "{{capture}}-p95", - "queryType": "randomWalk", - "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, - "interval": "", - "legendFormat": "{{lcapture}-p99}", -======= "intervalFactor": 1, "legendFormat": "approximate current time (s)", "refId": "A" @@ -1425,7 +1290,6 @@ "interval": "", "intervalFactor": 1, "legendFormat": "{{changefeed}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "B" }, { @@ -1459,12 +1323,7 @@ }, "yaxes": [ { -<<<<<<< HEAD - "$$hashKey": "object:1612", - "format": "s", -======= "format": "dateTimeAsIso", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -1472,7 +1331,6 @@ "show": true }, { - "$$hashKey": "object:1613", "format": "short", "label": null, "logBase": 1, @@ -1496,17 +1354,10 @@ "fill": 1, "fillGradient": 0, "gridPos": { -<<<<<<< HEAD - "h": 8, - "w": 12, - "x": 0, - "y": 18 -======= "h": 7, "w": 8, "x": 9, "y": 13 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "hiddenSeries": false, "id": 102, @@ -1599,17 +1450,10 @@ "description": "The number of errors that interrupt changefeed per minute ", "fill": 1, "gridPos": { -<<<<<<< HEAD - "h": 8, - "w": 12, - "x": 12, - "y": 18 -======= "h": 7, "w": 7, "x": 17, "y": 13 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "id": 82, "legend": { @@ -1664,12 +1508,7 @@ }, "yaxes": [ { -<<<<<<< HEAD - "$$hashKey": "object:2055", - "format": "bytes", -======= "format": "short", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -1677,7 +1516,6 @@ "show": true }, { - "$$hashKey": "object:2056", "format": "short", "label": null, "logBase": 1, @@ -1690,133 +1528,6 @@ "align": false, "alignLevel": null } -<<<<<<< HEAD - } - ], - "title": "EtcdWorker", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 266, - "panels": [ - { - "cards": { - "cardPadding": 1, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": null, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 2 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 262, - "legend": { - "show": true - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, capture)", - "format": "heatmap", - "interval": "1", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "EtcdWorker tick reactor duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "tooltipDecimals": null, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 2 - }, - "hiddenSeries": false, - "id": 264, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "6.1.6", -======= }, { "aliasColors": {}, @@ -1855,7 +1566,6 @@ }, "paceLength": 10, "percentage": false, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "pointradius": 2, "points": false, "renderer": "flot", @@ -1865,21 +1575,6 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "interval": "", - "legendFormat": "{{capture}}-95", - "queryType": "randomWalk", - "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, - "interval": "", - "legendFormat": "{{capture}}-99", -======= "expr": "max(ticdc_owner_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed)", "format": "time_series", "interval": "", @@ -2488,7 +2183,6 @@ "interval": "", "intervalFactor": 1, "legendFormat": "total", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "B" } ], @@ -2496,11 +2190,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "EtcdWorker tick reactor duration", -======= "title": "Sink write rows count/s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -2516,12 +2206,7 @@ }, "yaxes": [ { -<<<<<<< HEAD - "$$hashKey": "object:2612", - "format": "s", -======= "format": "none", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -2529,8 +2214,7 @@ "show": true }, { - "$$hashKey": "object:2613", - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2544,113 +2228,10 @@ } }, { -<<<<<<< HEAD - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": 1, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 256, - "legend": { - "show": true - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "EtcdWorker exec etcd txn duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "hiddenSeries": false, - "id": 258, - "legend": { - "avg": false, - "current": false, -======= - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "Percentiles of sink batch size", "fill": 1, @@ -2667,21 +2248,22 @@ "alignAsTable": true, "avg": false, "current": true, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { - "alertThreshold": true + "dataLinks": [] }, + "paceLength": 10, "percentage": false, - "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", @@ -2691,23 +2273,6 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "format": "time_series", - "interval": "", - "legendFormat": "{{capture}}-p95", - "queryType": "randomWalk", - "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, - "interval": "", - "legendFormat": "{{capture}-p99}", - "refId": "B" -======= "expr": "histogram_quantile(0.90, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", "intervalFactor": 1, @@ -2728,18 +2293,13 @@ "intervalFactor": 1, "legendFormat": "{{capture}}-p999", "refId": "C" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "EtcdWorker exec etcd txn duration percentile", -======= "title": "Sink write batch size percentile", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -2755,12 +2315,7 @@ }, "yaxes": [ { -<<<<<<< HEAD - "$$hashKey": "object:1612", - "format": "s", -======= "format": "none", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 2, "max": null, @@ -2768,7 +2323,6 @@ "show": true }, { - "$$hashKey": "object:1613", "format": "short", "label": null, "logBase": 1, @@ -2784,16 +2338,6 @@ }, { "cards": { -<<<<<<< HEAD - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolatePurples", - "exponent": 0.5, -======= "cardPadding": 0, "cardRound": 0 }, @@ -2803,57 +2347,21 @@ "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "min": 0, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 -======= "description": "Asynchronous flush sink duration of changefeeds", "gridPos": { "h": 7, "w": 12, "x": 0, "y": 48 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, -<<<<<<< HEAD - "id": 254, - "legend": { - "show": true - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, capture)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "EtcdWorker txn size ", -======= "id": 93, "legend": { "alignAsTable": true, @@ -2881,16 +2389,11 @@ } ], "title": "Flush sink duration", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "show": true, "showHistogram": true }, -<<<<<<< HEAD - "tooltipDecimals": null, -======= "tooltipDecimals": 1, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "type": "heatmap", "xAxis": { "show": true @@ -2898,24 +2401,15 @@ "xBucketNumber": null, "xBucketSize": null, "yAxis": { -<<<<<<< HEAD - "decimals": null, - "format": "decbytes", -======= "decimals": 1, "format": "s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "logBase": 1, "max": null, "min": null, "show": true, "splitFactor": null }, -<<<<<<< HEAD - "yBucketBound": "auto", -======= "yBucketBound": "upper", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "yBucketNumber": null, "yBucketSize": null }, @@ -2925,23 +2419,6 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 18 - }, - "hiddenSeries": false, - "id": 260, -======= "description": "Percentiles of asynchronous flush sink duration of changefeeds", "fill": 1, "fillGradient": 0, @@ -2953,30 +2430,26 @@ }, "hiddenSeries": false, "id": 98, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { -<<<<<<< HEAD - "alertThreshold": true - }, -======= "dataLinks": [] }, "paceLength": 10, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "percentage": false, - "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", @@ -2986,22 +2459,6 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "interval": "", - "legendFormat": "{{capture}}-p95", - "queryType": "randomWalk", - "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, - "interval": "", - "legendFormat": "{{capture}}-p99", - "refId": "B" -======= "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", "format": "time_series", "intervalFactor": 1, @@ -3021,18 +2478,13 @@ "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}-p999", "refId": "C" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "EtcdWorker txn size percentile", -======= "title": "Flush sink duration percentile", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -3048,12 +2500,7 @@ }, "yaxes": [ { -<<<<<<< HEAD - "$$hashKey": "object:2055", - "format": "bytes", -======= "format": "s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 2, "max": null, @@ -3061,7 +2508,6 @@ "show": true }, { - "$$hashKey": "object:2056", "format": "short", "label": null, "logBase": 1, @@ -3074,38 +2520,6 @@ "align": false, "alignLevel": null } -<<<<<<< HEAD - } - ], - "title": "EtcdWorker", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 11, - "panels": [ - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of captured table of TiCDC nodes ", - "fontSize": "100%", - "gridPos": { - "h": 5, - "w": 7, - "x": 0, - "y": 2 -======= }, { "cards": { @@ -3214,73 +2628,38 @@ "sortDesc": true, "total": false, "values": true ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "id": 4, + "lines": true, + "linewidth": 1, "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true + "nullPointMode": "null", + "options": { + "dataLinks": [] }, -<<<<<<< HEAD - "styles": [ + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(ticdc_processor_num_of_tables{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", -======= - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture,bucket)", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{capture}}-{{bucket}}", - "refId": "A" + "expr": "sum(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture,bucket)", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{capture}}-{{bucket}}", + "refId": "A" }, { "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) >= 0)", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "format": "time_series", "hide": true, "interval": "", "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{capture}}", - "refId": "A" -======= "legendFormat": "total worker", "refId": "B" }, @@ -3318,19 +2697,12 @@ "intervalFactor": 1, "legendFormat": ">100 row/s worker", "refId": "F" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], + "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Changefeed table count", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "columns": [ -======= "title": "MySQL sink worker load", "tooltip": { "shared": true, @@ -3354,22 +2726,15 @@ "min": null, "show": true }, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) { - "text": "Current", - "value": "current" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ], -<<<<<<< HEAD - "datasource": "${DS_TEST-CLUSTER}", - "description": "Internal resolved ts of TiCDC nodes", - "fontSize": "100%", - "gridPos": { - "h": 10, - "w": 7, - "x": 7, - "y": 2 -======= "yaxis": { "align": false, "alignLevel": null @@ -3480,68 +2845,14 @@ "show": true, "total": false, "values": true ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "id": 90, + "lines": true, + "linewidth": 1, "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true + "nullPointMode": "null", + "options": { + "dataLinks": [] }, -<<<<<<< HEAD - "styles": [ - { - "alias": "table", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "Metric", - "thresholds": [], - "type": "string", - "unit": "short" - }, - { - "alias": "resolved ts", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "MM-DD HH:mm:ss.SSS", - "decimals": 2, - "pattern": "Current", - "thresholds": [], - "type": "date", - "unit": "short" - } - ], - "targets": [ - { - "expr": "max(ticdc_processor_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}", - "refId": "A" - }, - { - "expr": "max(ticdc_processor_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed) > 0", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "checkpoint-{{capture}}-{{changefeed}}", - "refId": "B" -======= "percentage": false, "pointradius": 2, "points": false, @@ -3574,19 +2885,12 @@ "intervalFactor": 1, "legendFormat": "{{instance}}-p999", "refId": "C" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], + "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Processor resolved ts", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "columns": [ -======= "title": "MySQL sink conflict detect duration percentile", "tooltip": { "shared": true, @@ -3610,22 +2914,26 @@ "min": null, "show": true }, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) { - "text": "Current", - "value": "current" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Internal resolved ts of captured tables", - "fontSize": "100%", - "gridPos": { - "h": 10, - "w": 10, - "x": 14, - "y": 2 -======= "fill": 1, "gridPos": { "h": 7, @@ -3642,18 +2950,10 @@ "show": true, "total": false, "values": false ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "id": 30, + "lines": true, + "linewidth": 1, "links": [], -<<<<<<< HEAD - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true -======= "nullPointMode": "null", "percentage": false, "pointradius": 2, @@ -3689,55 +2989,25 @@ "name": null, "show": true, "values": [] ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, - "styles": [ + "yaxes": [ { -<<<<<<< HEAD - "alias": "table", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "Metric", - "thresholds": [], - "type": "string", - "unit": "short" -======= "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { - "alias": "resolved ts", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "MM-DD HH:mm:ss.SSS", - "decimals": 2, - "pattern": "Current", - "thresholds": [], - "type": "date", - "unit": "short" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ], -<<<<<<< HEAD - "targets": [ - { - "expr": "bottomk(10, max(ticdc_processor_table_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed,table))", -======= "yaxis": { "align": false, "alignLevel": null @@ -3789,23 +3059,9 @@ "targets": [ { "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{capture}}-{{changefeed}}-{{table}}", - "refId": "A" - }, - { - "expr": "bottomk(10, max(ticdc_processor_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed,table) > 0)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "checkpoint-{{capture}}-{{changefeed}}", - "refId": "B" -======= "legendFormat": "{{instance}}-p95", "refId": "A" }, @@ -3824,104 +3080,27 @@ "intervalFactor": 1, "legendFormat": "{{instance}}-p999", "refId": "C" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], + "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Table resolved ts", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of replicated tables maintained in owner", - "fontSize": "100%", - "gridPos": { - "h": 5, - "w": 7, - "x": 0, - "y": 7 + "title": "MySQL sink ddl execution duration percentile", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" }, - "id": 138, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": null, - "desc": false + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(ticdc_owner_maintain_table_num{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\",type=\"total\"}) by (capture)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{capture}}-total", - "refId": "A" - }, - { - "expr": "sum(ticdc_owner_maintain_table_num{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\",type=\"wip\"}) by (capture)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{capture}}-wip", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Table count maintained by owner", - "transform": "timeseries_aggregations", - "type": "table" -======= - "title": "MySQL sink ddl execution duration percentile", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ + "yaxes": [ { "format": "s", "label": null, @@ -3943,26 +3122,13 @@ "align": false, "alignLevel": null } ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "aliasColors": {}, - "bars": true, - "cacheTimeout": null, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The checkpoint ts of changefeeds.", - "fill": 0, - "gridPos": { - "h": 7, - "w": 9, - "x": 0, - "y": 12 - }, - "id": 86, -======= "fill": 1, "gridPos": { "h": 8, @@ -3971,75 +3137,41 @@ "y": 76 }, "id": 164, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "avg": false, "current": false, "max": false, "min": false, -<<<<<<< HEAD - "rightSide": false, -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, - "sideWidth": null, "total": false, "values": false }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/approximate current time.*/", - "bars": false - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "max(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed) > 0", -======= "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture, changefeed)", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "format": "time_series", - "interval": "", - "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{changefeed}}", - "refId": "A" - }, - { - "expr": "max(pd_cluster_tso{tidb_cluster=\"$tidb_cluster\"}) * 1000", "format": "time_series", - "hide": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "approximate current time (s)", - "refId": "B" -======= "legendFormat": "{{ capture }}-{{ changefeed }}", "refId": "A" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Changefeed checkpoint", -======= "title": "Processor Memory Consumption Per Changefeed", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4048,7 +3180,6 @@ "type": "graph", "xaxis": { "buckets": null, - "max": null, "mode": "time", "name": null, "show": true, @@ -4056,11 +3187,7 @@ }, "yaxes": [ { -<<<<<<< HEAD - "format": "dateTimeAsIso", -======= "format": "bytes", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -4073,7 +3200,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -4254,21 +3381,6 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Request count of etcd operation per second", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 9, - "y": 12 - }, - "hiddenSeries": false, - "id": 102, - "legend": { - "alignAsTable": true, -======= "description": "The number of events that puller outputs to sorter \n per second", "fill": 1, "fillGradient": 0, @@ -4282,11 +3394,8 @@ "id": 218, "legend": { "alignAsTable": false, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, "max": true, "min": false, "rightSide": false, @@ -4312,18 +3421,10 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(rate(ticdc_etcd_request_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (capture, type)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{capture}}-{{type}}", -======= "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture, type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -4331,11 +3432,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "PD etcd requests/s", -======= "title": "Puller output events/s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4351,7 +3448,7 @@ }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -4378,21 +3475,10 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of errors that interrupt changefeed per minute ", -======= "description": "The total number of events that puller outputs", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, + "fillGradient": 0, "gridPos": { -<<<<<<< HEAD - "h": 7, - "w": 7, - "x": 17, - "y": 12 - }, - "id": 82, -======= "h": 4, "w": 12, "x": 12, @@ -4400,7 +3486,6 @@ }, "hiddenSeries": false, "id": 229, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "legend": { "alignAsTable": false, "avg": false, @@ -4416,6 +3501,10 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -4426,17 +3515,10 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(delta(ticdc_processor_exit_with_error_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}", -======= "expr": "sum(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture, type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -4444,11 +3526,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Exit error count/m", -======= "title": "Puller output events", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -4464,7 +3542,7 @@ }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -4486,30 +3564,29 @@ } }, { -<<<<<<< HEAD "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.", + "description": "The number of events that sorter outputs to puller \n per second", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 0, - "y": 19 + "y": 8 }, "hiddenSeries": false, - "id": 3, + "id": 228, "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, - "max": false, + "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -4532,27 +3609,18 @@ "steppedLine": false, "targets": [ { - "expr": "max(ticdc_owner_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed)", + "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture, type)", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{changefeed}}", + "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", "refId": "A" - }, - { - "expr": "sum(ticdc_processor_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture,changefeed)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{capture}}", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Changefeed checkpoint lag", + "title": "Sorter output events/s", "tooltip": { "shared": true, "sort": 0, @@ -4568,11 +3636,11 @@ }, "yaxes": [ { - "format": "s", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -4581,7 +3649,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -4590,47 +3658,27 @@ } }, { -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "aliasColors": {}, - "bars": true, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The derivative of a changefeed checkpoint, ideally it should not be 0 (means the changefeed is stopped).", -======= - "description": "The number of events that sorter outputs to puller \n per second", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "description": "The total number of events that sorter outputs", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, -<<<<<<< HEAD "x": 12, - "y": 19 - }, - "hiddenSeries": false, - "id": 2, -======= - "x": 0, "y": 8 }, "hiddenSeries": false, - "id": 228, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 220, "legend": { "alignAsTable": false, "avg": false, "current": true, -<<<<<<< HEAD - "hideEmpty": false, - "hideZero": false, - "max": false, -======= "max": true, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "min": false, "rightSide": false, "show": true, @@ -4646,7 +3694,7 @@ }, "paceLength": 10, "percentage": false, - "pointradius": 0.5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -4655,19 +3703,10 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(deriv(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed) / 1000 > 0", -======= - "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture, type)", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "expr": "sum(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, type)", "format": "time_series", - "interval": "", "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{changefeed}}", -======= "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -4675,11 +3714,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Changefeed checkpoint derivative", -======= - "title": "Sorter output events/s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Sorter output events", "tooltip": { "shared": true, "sort": 0, @@ -4695,25 +3730,20 @@ }, "yaxes": [ { -<<<<<<< HEAD - "format": "s", + "format": "none", "label": null, - "logBase": 2, - "max": "3600", - "min": "0.001", + "logBase": 1, + "max": null, + "min": null, "show": true }, { "format": "short", -======= - "format": "none", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, "min": null, "show": true -<<<<<<< HEAD } ], "yaxis": { @@ -4727,21 +3757,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The status of each changefeed.\n\n0: Normal\n\n1: Error\n\n2: Failed\n\n3: Stopped\n\n4: Finished\n\n-1: Unknown", + "description": "The number of events that mounter outputs to sink per second", "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 0, - "y": 26 + "y": 12 }, - "id": 163, + "hiddenSeries": false, + "id": 219, "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, "max": true, - "min": true, + "min": false, + "rightSide": false, "show": true, "total": false, "values": true @@ -4750,9 +3783,13 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, "percentage": false, - "pointradius": 1, - "points": true, + "pointradius": 2, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -4760,11 +3797,10 @@ "steppedLine": false, "targets": [ { - "expr": "ticdc_owner_status{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}", + "expr": "sum(rate(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", "format": "time_series", - "instant": false, "intervalFactor": 1, - "legendFormat": "{{changefeed}}", + "legendFormat": "{{changefeed}}-{{capture}}", "refId": "A" } ], @@ -4772,7 +3808,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "The status of changefeeds", + "title": "Mounter output events/s", "tooltip": { "shared": true, "sort": 0, @@ -4788,14 +3824,12 @@ }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "format": "short", @@ -4803,11 +3837,7 @@ "logBase": 1, "max": null, "min": null, -<<<<<<< HEAD - "show": true -======= "show": false ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "yaxis": { @@ -4817,59 +3847,38 @@ }, { "aliasColors": {}, - "bars": true, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Estimate the remaining time for a changefeed catch-up upstream TiDB.", - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, -======= - "description": "The total number of events that sorter outputs", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "description": "The total number of events that mounter outputs", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 12, -<<<<<<< HEAD - "y": 26 + "y": 12 }, "hiddenSeries": false, - "id": 253, - "legend": { - "alignAsTable": true, -======= - "y": 8 - }, - "hiddenSeries": false, - "id": 220, + "id": 224, "legend": { "alignAsTable": false, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, + "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true }, - "lines": false, + "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { - "alertThreshold": true + "dataLinks": [] }, "paceLength": 10, "percentage": false, @@ -4882,19 +3891,10 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "exemplar": true, - "expr": "abs(max(ticdc_owner_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"} / (deriv(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])/1000)) by (changefeed))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{changefeed}}", -======= - "expr": "sum(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, type)", + "expr": "sum(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{capture}}-{{type}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{changefeed}}-{{capture}}", "refId": "A" } ], @@ -4902,11 +3902,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Changefeed checkpoint catch-up ETA", -======= - "title": "Sorter output events", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Mounter output events", "tooltip": { "shared": true, "sort": 0, @@ -4922,19 +3918,11 @@ }, "yaxes": [ { -<<<<<<< HEAD - "format": "s", - "label": "", - "logBase": 2, - "max": "604800", - "min": "1", -======= "format": "none", "label": null, "logBase": 1, "max": null, "min": null, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true }, { @@ -4957,30 +3945,17 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Sink write duration of changefeeds", -======= - "description": "The number of events that mounter outputs to sink per second", + "description": "The number of events that table sorter outputs to buffer sink per second", "fill": 1, "fillGradient": 0, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "gridPos": { "h": 4, "w": 12, "x": 0, -<<<<<<< HEAD - "y": 33 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 94, -======= - "y": 12 + "y": 16 }, "hiddenSeries": false, - "id": 219, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 223, "legend": { "alignAsTable": false, "avg": false, @@ -4995,22 +3970,6 @@ "lines": true, "linewidth": 1, "links": [], -<<<<<<< HEAD - "repeat": null, - "repeatDirection": "h", - "reverseYBuckets": false, - "targets": [ - { - "expr": "max(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Sink write duration", -======= "nullPointMode": "null", "options": { "dataLinks": [] @@ -5026,7 +3985,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", + "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{capture}}", @@ -5037,8 +3996,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Mounter output events/s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Table sink output events/s", "tooltip": { "shared": true, "sort": 0, @@ -5081,35 +4039,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Percentiles of sink write duration of changefeeds", -======= - "description": "The total number of events that mounter outputs", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "description": "The total number of events that table sinks emit", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 12, -<<<<<<< HEAD - "y": 33 - }, - "hiddenSeries": false, - "id": 35, -======= - "y": 12 + "y": 16 }, "hiddenSeries": false, - "id": 224, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 221, "legend": { "alignAsTable": false, "avg": false, "current": true, - "max": false, + "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -5132,43 +4079,18 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", -======= - "expr": "sum(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "expr": "sum(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{capture}}", "refId": "A" -<<<<<<< HEAD - }, - { - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-p99", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-p999", - "refId": "C" -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Sink write duration percentile", -======= - "title": "Mounter output events", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Table sink output events", "tooltip": { "shared": true, "sort": 0, @@ -5211,25 +4133,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of changed rows that are written to downstream per second", + "description": "The total number of events that buffer sink outputs to backend sink per second", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 0, - "y": 40 + "y": 20 }, "hiddenSeries": false, - "id": 34, + "id": 222, "legend": { - "alignAsTable": true, - "avg": true, + "alignAsTable": false, + "avg": false, "current": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -5252,27 +4173,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum (rate(ticdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", + "expr": "sum(rate(ticdc_sink_buffer_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}", + "legendFormat": "{{changefeed}}-{{capture}}", "refId": "A" - }, - { - "expr": "sum (rate(ticdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "total", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Sink write rows count/s", + "title": "Buffer sink output events/s", "tooltip": { "shared": true, "sort": 0, @@ -5301,7 +4213,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -5315,36 +4227,17 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Percentiles of sink batch size", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 40 - }, - "hiddenSeries": false, - "id": 36, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, -======= - "description": "The number of events that table sorter outputs to buffer sink per second", + "description": "The total number of events that buffer sink emits", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 0, - "y": 16 + "x": 12, + "y": 20 }, "hiddenSeries": false, - "id": 223, + "id": 226, "legend": { "alignAsTable": false, "avg": false, @@ -5352,7 +4245,6 @@ "max": true, "min": false, "rightSide": false, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, "total": false, "values": true @@ -5375,44 +4267,18 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "histogram_quantile(0.90, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}-p90", -======= - "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", + "expr": "sum(ticdc_sink_buffer_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{capture}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}-p99", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "{{capture}}-p999", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Sink write batch size percentile", -======= - "title": "Table sink output events/s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Buffer sink output events", "tooltip": { "shared": true, "sort": 0, @@ -5430,7 +4296,7 @@ { "format": "none", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -5441,7 +4307,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -5450,81 +4316,100 @@ } }, { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Asynchronous flush sink duration of changefeeds", + "description": "The number of rows(events) that are processed by sink per second", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 0, - "y": 47 + "y": 24 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 93, + "hiddenSeries": false, + "id": 227, "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, + "lines": true, + "linewidth": 1, "links": [], - "reverseYBuckets": false, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "max(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", + "expr": "sum (rate(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{capture}}", "refId": "A" } ], - "title": "Flush sink duration", + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sink output events/s", "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true + "shared": true, + "sort": 2, + "value_type": "individual" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -5532,36 +4417,27 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Percentiles of asynchronous flush sink duration of changefeeds", -======= - "description": "The total number of events that table sinks emit", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "description": "The number of rows(events) that are processed by sink", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 12, -<<<<<<< HEAD - "y": 47 - }, - "hiddenSeries": false, - "id": 98, -======= - "y": 16 + "y": 24 }, "hiddenSeries": false, - "id": 221, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 225, "legend": { "alignAsTable": false, "avg": false, "current": true, - "max": false, + "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -5572,7 +4448,6 @@ "options": { "dataLinks": [] }, - "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -5583,47 +4458,22 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}-p95", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}-p99", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}-p999", - "refId": "C" -======= - "expr": "sum(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", + "expr": "sum(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", "format": "time_series", + "interval": "", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{capture}}", "refId": "A" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Flush sink duration percentile", -======= - "title": "Table sink output events", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Sink output events", "tooltip": { "shared": true, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -5657,121 +4507,26 @@ "alignLevel": null } }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": 1, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The duration of detecting and waiting conflict of MySQL sink", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 54 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 103, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "repeatDirection": "h", - "reverseYBuckets": false, - "targets": [ - { - "expr": "max(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "MySQL sink conflict detect duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Distribution of MySQL worker loads", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 54 - }, - "hiddenSeries": false, - "id": 95, - "legend": { - "alignAsTable": true, - "avg": true, -======= - "description": "The total number of events that buffer sink outputs to backend sink per second", + "description": "The number of rows that sink flushes to downstream per second", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 0, - "y": 20 + "y": 28 }, "hiddenSeries": false, - "id": 222, + "id": 108, "legend": { "alignAsTable": false, "avg": false, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "current": true, "max": true, "min": false, @@ -5789,90 +4544,28 @@ "options": { "dataLinks": [] }, -<<<<<<< HEAD -======= - "paceLength": 10, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture,bucket)", -======= - "expr": "sum(rate(ticdc_sink_buffer_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "expr": "sum(rate(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", "format": "time_series", - "hide": true, - "interval": "", "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{capture}}-{{bucket}}", -======= "legendFormat": "{{changefeed}}-{{capture}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" - }, - { - "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) >= 0)", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "total worker", - "refId": "B" - }, - { - "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) <= 2)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "0-2 row/s worker", - "refId": "C" - }, - { - "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) > 2 and rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) <= 10)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "2-10 row/s worker", - "refId": "D" - }, - { - "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) > 10 and rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) <= 100)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "10-100 row/s worker", - "refId": "E" - }, - { - "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) > 100)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": ">100 row/s worker", - "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "MySQL sink worker load", -======= - "title": "Buffer sink output events/s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Sink flush rows/s", "tooltip": { "shared": true, "sort": 2, @@ -5888,11 +4581,7 @@ }, "yaxes": [ { -<<<<<<< HEAD - "format": "short", -======= "format": "none", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -5900,7 +4589,7 @@ "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -5914,110 +4603,35 @@ } }, { -<<<<<<< HEAD - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": 1, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The duration of executing DDL in MySQL sink", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 61 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 165, - "legend": { - "alignAsTable": true, -======= "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The total number of events that buffer sink emits", + "description": "The number of rows(events) that are flushed by sink", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 12, - "y": 20 + "y": 28 }, "hiddenSeries": false, - "id": 226, + "id": 240, "legend": { "alignAsTable": false, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "avg": false, "current": true, "max": true, "min": false, -<<<<<<< HEAD - "rightSide": true, + "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, - "links": [], - "repeatDirection": "h", - "reverseYBuckets": false, - "targets": [ - { - "expr": "max(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "MySQL sink ddl execution duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null -======= - "rightSide": false, - "show": true, - "total": false, - "values": true - }, "lines": true, "linewidth": 1, "links": [], @@ -6025,7 +4639,6 @@ "options": { "dataLinks": [] }, - "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -6036,8 +4649,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_sink_buffer_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", + "expr": "sum(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed, capture)", "format": "time_series", + "interval": "", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{capture}}", "refId": "A" @@ -6047,10 +4661,10 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Buffer sink output events", + "title": "Sink flush rows", "tooltip": { "shared": true, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -6083,47 +4697,47 @@ "align": false, "alignLevel": null } ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - }, + } + ], + "title": "Dataflow", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 13, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Percentiles of detecting and waiting conflict duration of MySQL sink", -======= - "description": "The number of rows(events) that are processed by sink per second", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "description": "The number of established Eventfeed RPC between TiCDC and TiKV", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 12, -<<<<<<< HEAD - "x": 12, - "y": 61 - }, - "hiddenSeries": false, - "id": 83, -======= + "w": 8, "x": 0, - "y": 24 + "y": 5 }, "hiddenSeries": false, - "id": 227, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 15, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": true, + "hideEmpty": false, + "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, @@ -6134,60 +4748,60 @@ "options": { "dataLinks": [] }, + "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*-rpc/", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-p95", -======= - "expr": "sum (rate(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", + "expr": "sum(ticdc_kvclient_event_feed_count{tidb_cluster=\"$tidb_cluster\"}) by (instance)", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{capture}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{instance}}", "refId": "A" }, { - "expr": "histogram_quantile(0.99,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "expr": "sum(grpc_client_started_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance) - sum(grpc_client_handled_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-p99", + "legendFormat": "{{instance}}-rpc", "refId": "B" }, { - "expr": "histogram_quantile(0.999,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "expr": "sum(grpc_client_started_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", "format": "time_series", - "interval": "", + "hide": false, "intervalFactor": 1, - "legendFormat": "{{instance}}-p999", + "legendFormat": "{{instance}}-rpc-started", "refId": "C" + }, + { + "expr": "sum(grpc_client_handled_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}-rpc-handled", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "MySQL sink conflict detect duration percentile", -======= - "title": "Sink output events/s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Eventfeed count", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -6200,9 +4814,9 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -6227,47 +4841,37 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD -======= - "description": "The number of rows(events) that are processed by sink", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "decimals": null, + "description": "Percentiles of Eventfeed message size", "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 8, - "w": 12, -<<<<<<< HEAD - "x": 0, - "y": 68 - }, - "id": 149, -======= - "x": 12, - "y": 24 + "h": 7, + "w": 8, + "x": 8, + "y": 5 }, "hiddenSeries": false, - "id": 225, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 17, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", -<<<<<<< HEAD -======= "options": { "dataLinks": [] }, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -6278,34 +4882,28 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture)", -======= - "expr": "sum(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (changefeed, capture)", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "expr": "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", "format": "time_series", - "interval": "", "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{ capture }}", -======= - "legendFormat": "{{changefeed}}-{{capture}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{instance}}-{{type}}-p999", "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p95", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Processor Memory Consumption Per Capture", -======= - "title": "Sink output events", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Event size percentile", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -6341,43 +4939,31 @@ }, { "aliasColors": {}, - "bars": false, + "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Percentiles of executing DDL duration of MySQL sink", -======= - "description": "The number of rows that sink flushes to downstream per second", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "description": "The number of errors that interrupt Eventfeed RPC", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 12, -<<<<<<< HEAD - "x": 12, - "y": 68 - }, - "hiddenSeries": false, - "id": 166, -======= - "x": 0, - "y": 28 + "w": 8, + "x": 16, + "y": 5 }, "hiddenSeries": false, - "id": 108, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 28, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, @@ -6388,6 +4974,7 @@ "options": { "dataLinks": [] }, + "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -6398,35 +4985,29 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", -======= - "expr": "sum(rate(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (changefeed, capture)", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "expr": "sum(increase(ticdc_kvclient_event_feed_error_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (type)", "format": "time_series", - "interval": "", + "hide": false, + "interval": "1m", "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{instance}}-p95", -======= - "legendFormat": "{{changefeed}}-{{capture}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{type}}", "refId": "A" }, { - "expr": "histogram_quantile(0.99,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "expr": "-sum(increase(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))", "format": "time_series", - "interval": "", + "hide": false, + "interval": "1m", "intervalFactor": 1, - "legendFormat": "{{instance}}-p99", + "legendFormat": "transfer-leader", "refId": "B" }, { - "expr": "histogram_quantile(0.999,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", + "expr": "-sum(increase(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))", "format": "time_series", - "interval": "", + "interval": "1m", "intervalFactor": 1, - "legendFormat": "{{instance}}-p999", + "legendFormat": "move-region", "refId": "C" } ], @@ -6434,14 +5015,10 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "MySQL sink ddl execution duration percentile", -======= - "title": "Sink flush rows/s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Eventfeed error/m", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -6454,15 +5031,15 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -6481,46 +5058,38 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD -======= - "description": "The number of rows(events) that are flushed by sink", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "description": "The number of KV client received events from TiKV per seconds", "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 8, - "w": 12, - "x": 12, -<<<<<<< HEAD - "y": 75 - }, - "id": 164, -======= - "y": 28 + "h": 7, + "w": 8, + "x": 0, + "y": 12 }, "hiddenSeries": false, - "id": 240, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 29, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, "min": false, + "rightSide": false, "show": true, - "sort": "current", - "sortDesc": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", -<<<<<<< HEAD -======= "options": { "dataLinks": [] }, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -6531,19 +5100,10 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture, changefeed)", -======= - "expr": "sum(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed, capture)", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "expr": "sum(rate(ticdc_kvclient_pull_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", "format": "time_series", - "interval": "", "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{ capture }}-{{ changefeed }}", -======= - "legendFormat": "{{changefeed}}-{{capture}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{instance}}-{{type}}", "refId": "A" } ], @@ -6551,14 +5111,10 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Processor Memory Consumption Per Changefeed", -======= - "title": "Sink flush rows", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "KV client receive events/s", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -6571,7 +5127,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -6591,224 +5147,31 @@ "align": false, "alignLevel": null } - } - ], - "title": "Dataflow", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 13, - "panels": [ - { - "columns": [], - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "left", - "filterable": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "dateTimeAsSystem" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "capture" - }, - "properties": [ - { - "id": "custom.width", - "value": 191 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "changefeed" - }, - "properties": [ - { - "id": "custom.width", - "value": 209 - } - ] - } - ] - }, - "fontSize": "100%", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 200, - "options": { - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "last-snap-ts" - } - ] - }, - "pageSize": null, - "pluginVersion": "6.1.6", - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "exemplar": true, - "expr": "ticdc_processor_schema_storage_gc_ts{changefeed=~\"$changefeed\", capture=~\"$capture\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Schema Storage GC progress", - "transform": "table", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "capture", - "changefeed", - "Value" - ] - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "Value #A": "last-snap-ts" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "last-snap-ts" - } - ] - } - } - ], - "type": "table" - } - ], - "title": "Changefeed", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 2 - }, - "id": 206, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD "description": "The number of events that puller outputs to sorter \n per second", -======= - "description": "The number of established Eventfeed RPC between TiCDC and TiKV", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, - "x": 0, -<<<<<<< HEAD - "y": 3 - }, - "hiddenSeries": false, - "id": 218, -======= - "y": 5 + "x": 8, + "y": 12 }, "hiddenSeries": false, - "id": 15, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 5, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, - "max": false, + "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -6825,63 +5188,24 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*-rpc/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { -<<<<<<< HEAD "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", -======= - "expr": "sum(ticdc_kvclient_event_feed_count{tidb_cluster=\"$tidb_cluster\"}) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" - }, - { - "expr": "sum(grpc_client_started_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance) - sum(grpc_client_handled_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-rpc", - "refId": "B" - }, - { - "expr": "sum(grpc_client_started_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}-rpc-started", - "refId": "C" - }, - { - "expr": "sum(grpc_client_handled_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}-rpc-handled", - "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD "title": "Puller output events/s", -======= - "title": "Eventfeed count", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "tooltip": { "shared": true, "sort": 0, @@ -6897,7 +5221,7 @@ }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -6924,39 +5248,25 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The total number of events that puller outputs", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 3 - }, - "hiddenSeries": false, - "id": 229, -======= - "decimals": null, - "description": "Percentiles of Eventfeed message size", + "description": "The number of events that are buffered in Processor's output channel and Mounter input channel", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, - "x": 8, - "y": 5 + "x": 16, + "y": 12 }, "hiddenSeries": false, - "id": 17, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 107, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "hideEmpty": false, + "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -6979,24 +5289,17 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance, type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}", -======= - "expr": "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", + "expr": "sum(ticdc_mounter_input_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}-p999", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{capture}}-mounter input chan", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", + "expr": "-sum(ticdc_sink_buffer_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}-p95", + "legendFormat": "{{capture}}-sink buffer chan", "refId": "B" } ], @@ -7004,11 +5307,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Puller output events", -======= - "title": "Event size percentile", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Sink/Mounter buffer size", "tooltip": { "shared": true, "sort": 0, @@ -7024,7 +5323,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -7046,42 +5345,103 @@ } }, { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of events that sorter outputs to puller \n per second", - "fill": 1, - "fillGradient": 0, + "description": "Duration of sorting unsorted events", "gridPos": { - "h": 4, + "h": 7, "w": 12, "x": 0, - "y": 7 + "y": 19 }, - "hiddenSeries": false, - "id": 228, -======= - "description": "The number of errors that interrupt Eventfeed RPC", + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Entry sorter sort duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Percentiles of sorting events duration", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 8, - "x": 16, - "y": 5 + "w": 12, + "x": 12, + "y": 19 }, "hiddenSeries": false, - "id": 28, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 53, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, + "hideEmpty": false, "max": true, "min": false, "rightSide": true, @@ -7107,50 +5467,26 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed,type)", -======= - "expr": "sum(increase(ticdc_kvclient_event_feed_error_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (type)", + "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", - "hide": false, - "interval": "1m", "intervalFactor": 1, - "legendFormat": "{{type}}", + "legendFormat": "{{capture}}-p999", "refId": "A" }, { - "expr": "-sum(increase(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))", + "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", "hide": false, - "interval": "1m", "intervalFactor": 1, - "legendFormat": "transfer-leader", + "legendFormat": "{{capture}}-p95", "refId": "B" - }, - { - "expr": "-sum(increase(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "format": "time_series", - "interval": "1m", - "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{capture}}-{{type}}", - "refId": "A" -======= - "legendFormat": "move-region", - "refId": "C" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Sorter output events/s", -======= - "title": "Eventfeed error/m", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Entry sorter sort duration percentile", "tooltip": { "shared": true, "sort": 0, @@ -7166,9 +5502,9 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -7188,244 +5524,80 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The total number of events that sorter outputs", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 7 + "cards": { + "cardPadding": 0, + "cardRound": 0 }, - "hiddenSeries": false, - "id": 220, -======= - "description": "The number of KV client received events from TiKV per seconds", - "fill": 1, - "fillGradient": 0, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "Duration of merging sorted events", "gridPos": { "h": 7, - "w": 8, + "w": 12, "x": 0, - "y": 12 + "y": 26 }, - "hiddenSeries": false, - "id": 29, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 105, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "reverseYBuckets": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (capture, changefeed, type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}-{{type}}", -======= - "expr": "sum(rate(ticdc_kvclient_pull_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, -<<<<<<< HEAD - "title": "Sorter output events", -======= - "title": "KV client receive events/s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Entry sorter merge duration", "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of events that mounter outputs to sink per second", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 11 - }, - "hiddenSeries": false, - "id": 219, -======= - "description": "The number of events that puller outputs to sorter \n per second", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 12 - }, - "hiddenSeries": false, - "id": 5, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": false, "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] + "showHistogram": true }, - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { -<<<<<<< HEAD - "expr": "sum(rate(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}", -======= - "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, -<<<<<<< HEAD - "title": "Mounter output events/s", -======= - "title": "Puller output events/s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null }, { "aliasColors": {}, @@ -7433,31 +5605,17 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The total number of events that mounter outputs", + "description": "Percentiles of merging sorted events duration", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 4, + "h": 7, "w": 12, "x": 12, - "y": 11 - }, - "hiddenSeries": false, - "id": 224, -======= - "description": "The number of events that are buffered in Processor's output channel and Mounter input channel", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 12 + "y": 26 }, "hiddenSeries": false, - "id": 107, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 106, "legend": { "alignAsTable": true, "avg": false, @@ -7465,7 +5623,7 @@ "hideEmpty": false, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -7488,21 +5646,18 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(ticdc_mounter_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance)", -======= - "expr": "sum(ticdc_mounter_input_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-mounter input chan", + "legendFormat": "{{capture}}-p999", "refId": "A" }, { - "expr": "-sum(ticdc_sink_buffer_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", + "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "{{capture}}-sink buffer chan", + "legendFormat": "{{capture}}-p95", "refId": "B" } ], @@ -7510,11 +5665,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Mounter output events", -======= - "title": "Sink/Mounter buffer size", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Entry sorter merge duration percentile", "tooltip": { "shared": true, "sort": 0, @@ -7530,9 +5681,9 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -7566,17 +5717,17 @@ }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "Duration of sorting unsorted events", + "description": "Duration of unmarshal events from kv to SQL row", "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 19 + "y": 33 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 99, + "id": 101, "legend": { "alignAsTable": true, "avg": false, @@ -7594,7 +5745,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "expr": "max(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -7602,7 +5753,7 @@ "refId": "A" } ], - "title": "Entry sorter sort duration", + "title": "Mounter unmarshal duration", "tooltip": { "show": true, "showHistogram": true @@ -7633,38 +5784,27 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of events that table sorter outputs to buffer sink per second", -======= - "description": "Percentiles of sorting events duration", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "fill": 1, + "description": "Percentiles of unmarshal events from kv to SQL row duration", + "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, -<<<<<<< HEAD - "x": 0, - "y": 15 - }, - "hiddenSeries": false, - "id": 223, -======= "x": 12, - "y": 19 + "y": 33 }, "hiddenSeries": false, - "id": 53, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 55, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, "max": true, "min": false, "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -7686,25 +5826,20 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", -======= - "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "histogram_quantile(0.99, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "{{capture}}-p999", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{capture}}-p99", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "histogram_quantile(0.999, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", "format": "time_series", - "hide": false, + "hide": true, + "instant": false, "intervalFactor": 1, - "legendFormat": "{{capture}}-p95", + "legendFormat": "{{capture}}-p999", "refId": "B" } ], @@ -7712,14 +5847,10 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Table sink output events/s", -======= - "title": "Entry sorter sort duration percentile", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Mounter unmarshal duration percentile", "tooltip": { "shared": true, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -7734,13 +5865,13 @@ { "format": "s", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -7753,115 +5884,29 @@ "alignLevel": null } }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "Duration of merging sorted events", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 26 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 105, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Entry sorter merge duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The total number of events that table sinks emit", -======= - "description": "Percentiles of merging sorted events duration", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "description": "The number of KV client dispatched event per second", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, - "x": 12, -<<<<<<< HEAD - "y": 15 - }, - "hiddenSeries": false, - "id": 221, -======= - "y": 26 + "x": 0, + "y": 40 }, "hiddenSeries": false, - "id": 106, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 31, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, "rightSide": true, @@ -7881,31 +5926,28 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*batch-resolved/", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(ticdc_sink_table_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", -======= - "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "sum(rate(ticdc_kvclient_send_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture, changefeed, type)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-p999", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{capture}}-{{changefeed}}-{{type}}", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture, changefeed, table)", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "{{capture}}-p95", + "legendFormat": "{{capture}}-{{changefeed}}-batch-resolved", "refId": "B" } ], @@ -7913,11 +5955,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Table sink output events", -======= - "title": "Entry sorter merge duration percentile", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "KV client dispatch events/s", "tooltip": { "shared": true, "sort": 0, @@ -7933,9 +5971,9 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true @@ -7969,30 +6007,17 @@ }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The total number of events that buffer sink outputs to backend sink per second", - "fill": 1, - "fillGradient": 0, -======= - "description": "Duration of unmarshal events from kv to SQL row", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "description": "The size of batch resolved ts message from TiKV", "gridPos": { "h": 7, "w": 12, - "x": 0, -<<<<<<< HEAD - "y": 19 - }, - "hiddenSeries": false, - "id": 222, -======= - "y": 33 + "x": 12, + "y": 40 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 101, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 97, "legend": { "alignAsTable": true, "avg": false, @@ -8001,47 +6026,16 @@ "min": false, "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "links": [], -<<<<<<< HEAD - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(ticdc_sink_buffer_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Buffer sink output events/s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" -======= "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -8049,11 +6043,10 @@ "refId": "A" } ], - "title": "Mounter unmarshal duration", + "title": "KV client batch resolved size", "tooltip": { "show": true, "showHistogram": true ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, "tooltipDecimals": 1, "type": "heatmap", @@ -8064,41 +6057,16 @@ "xBucketSize": null, "yAxis": { "decimals": 1, - "format": "s", + "format": "none", "logBase": 1, "max": null, "min": null, "show": true, "splitFactor": null }, -<<<<<<< HEAD - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } -======= "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "aliasColors": {}, @@ -8106,33 +6074,23 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The total number of events that buffer sink emits", + "description": "The number of regions that are being scanned", "fill": 1, -======= - "description": "Percentiles of unmarshal events from kv to SQL row duration", - "fill": 0, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fillGradient": 0, "gridPos": { "h": 7, "w": 12, - "x": 12, -<<<<<<< HEAD - "y": 19 - }, - "hiddenSeries": false, - "id": 226, -======= - "y": 33 + "x": 0, + "y": 47 }, "hiddenSeries": false, - "id": 55, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 177, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, "rightSide": true, @@ -8158,39 +6116,18 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(ticdc_sink_buffer_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", -======= - "expr": "histogram_quantile(0.99, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", + "expr": "sum(ticdc_kvclient_region_token{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "{{capture}}-p99", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", "refId": "A" - }, - { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", - "format": "time_series", - "hide": true, - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{capture}}-p999", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Buffer sink output events", -======= - "title": "Mounter unmarshal duration percentile", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "KV client scanning regions", "tooltip": { "shared": true, "sort": 0, @@ -8206,7 +6143,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -8214,11 +6151,7 @@ "show": true }, { -<<<<<<< HEAD "format": "short", -======= - "format": "none", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -8237,46 +6170,27 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of rows(events) that are processed by sink per second", + "description": "Active stream count of each gRPC connection", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 4, + "h": 7, "w": 12, - "x": 0, - "y": 23 + "x": 12, + "y": 47 }, "hiddenSeries": false, - "id": 227, -======= - "description": "The number of KV client dispatched event per second", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 40 - }, - "hiddenSeries": false, - "id": 31, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 188, "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, -<<<<<<< HEAD -======= "hideEmpty": true, "hideZero": true, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, @@ -8287,61 +6201,32 @@ "options": { "dataLinks": [] }, + "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", -<<<<<<< HEAD "seriesOverrides": [], -======= - "seriesOverrides": [ - { - "alias": "/.*batch-resolved/", - "yaxis": 2 - } - ], ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum (rate(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture)", -======= - "expr": "sum(rate(ticdc_kvclient_send_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture, changefeed, type)", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "expr": "sum(ticdc_kvclient_grpc_stream_count{tidb_cluster=\"$tidb_cluster\"}) by (store)", "format": "time_series", - "interval": "", "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{capture}}", - "refId": "A" -======= - "legendFormat": "{{capture}}-{{changefeed}}-{{type}}", + "legendFormat": "{{store}}", "refId": "A" - }, - { - "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture, changefeed, table)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}-batch-resolved", - "refId": "B" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Sink output events/s", -======= - "title": "KV client dispatch events/s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "KV client gRPC stream count", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -8354,7 +6239,7 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -8367,7 +6252,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -8375,128 +6260,39 @@ "alignLevel": null } }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The size of batch resolved ts message from TiKV", - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 40 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 97, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "KV client batch resolved size", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "none", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of rows(events) that are processed by sink", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 23 + "description": "The number of regions that have not connected to TiKV", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] }, - "hiddenSeries": false, - "id": 225, -======= - "description": "The number of regions that are being scanned", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 47 + "y": 54 }, "hiddenSeries": false, - "id": 177, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 251, "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, -<<<<<<< HEAD -======= "hideEmpty": true, "hideZero": true, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, @@ -8505,8 +6301,9 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -8517,19 +6314,12 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (capture, type)", -======= - "expr": "sum(ticdc_kvclient_region_token{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "exemplar": true, + "expr": "sum(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", "format": "time_series", "interval": "", "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{capture}}", -======= "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "refId": "A" } ], @@ -8537,14 +6327,10 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Sink output events", -======= - "title": "KV client scanning regions", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "KV client cached regions", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -8557,11 +6343,7 @@ }, "yaxes": [ { -<<<<<<< HEAD - "format": "none", -======= "format": "short", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -8588,41 +6370,33 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of rows that sink flushes to downstream per second", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 27 + "description": "Estimate the remaining time for a changefeed initialization (on a specific capture)", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] }, - "hiddenSeries": false, - "id": 108, -======= - "description": "Active stream count of each gRPC connection", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 47 + "y": 54 }, "hiddenSeries": false, - "id": 188, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 252, "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, @@ -8631,8 +6405,9 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, @@ -8643,17 +6418,12 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(rate(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}", -======= - "expr": "sum(ticdc_kvclient_grpc_stream_count{tidb_cluster=\"$tidb_cluster\"}) by (store)", + "exemplar": true, + "expr": "abs(sum(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"} / deriv(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture, changefeed, store))", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{store}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", "refId": "A" } ], @@ -8661,14 +6431,10 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Sink flush rows/s", -======= - "title": "KV client gRPC stream count", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Estimate remaining time for initialization", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -8681,84 +6447,68 @@ }, "yaxes": [ { - "format": "none", - "label": null, - "logBase": 1, + "format": "s", + "label": "", + "logBase": 2, "max": null, "min": null, "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { "align": false, "alignLevel": null } - }, + } + ], + "title": "Events", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 130, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of rows(events) that are flushed by sink", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 27 - }, - "hiddenSeries": false, - "id": 240, -======= - "description": "The number of regions that have not connected to TiKV", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 0, - "y": 54 + "y": 5 }, - "hiddenSeries": false, - "id": 251, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 131, "legend": { - "alignAsTable": false, "avg": false, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", - "options": { - "alertThreshold": true - }, "percentage": false, "pointradius": 2, "points": false, @@ -8769,20 +6519,10 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (capture, type)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{capture}}", -======= - "exemplar": true, - "expr": "sum(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", + "expr": "sum(rate(ticdc_sorter_consume_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{capture}}-{{changefeed}}", "refId": "A" } ], @@ -8790,14 +6530,10 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Sink flush rows", -======= - "title": "KV client cached regions", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Unified Sorter intake rate", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -8810,7 +6546,7 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -8830,147 +6566,56 @@ "align": false, "alignLevel": null } - } - ], - "title": "Dataflow", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 13, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of established Eventfeed RPC between TiCDC and TiKV", -======= - "description": "Estimate the remaining time for a changefeed initialization (on a specific capture)", - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 7, -<<<<<<< HEAD - "w": 8, - "x": 0, - "y": 32 - }, - "hiddenSeries": false, - "id": 15, -======= + "h": 8, "w": 12, "x": 12, - "y": 54 + "y": 5 }, - "hiddenSeries": false, - "id": 252, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 132, "legend": { - "alignAsTable": true, "avg": false, - "current": true, -<<<<<<< HEAD - "hideEmpty": false, + "current": false, "max": false, -======= - "hideEmpty": true, - "hideZero": true, - "max": true, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "min": false, - "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*-rpc/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(ticdc_kvclient_event_feed_count{tidb_cluster=\"$tidb_cluster\"}) by (instance)", -======= - "exemplar": true, - "expr": "abs(sum(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"} / deriv(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture, changefeed, store))", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", "format": "time_series", - "interval": "", "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{instance}}", -======= - "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{capture}}-{{changefeed}}", "refId": "A" - }, - { - "expr": "sum(grpc_client_started_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance) - sum(grpc_client_handled_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-rpc", - "refId": "B" - }, - { - "expr": "sum(grpc_client_started_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}-rpc-started", - "refId": "C" - }, - { - "expr": "sum(grpc_client_handled_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}-rpc-handled", - "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Eventfeed count", -======= - "title": "Estimate remaining time for initialization", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Unified Sorter event output rate", "tooltip": { "shared": true, "sort": 0, @@ -8986,15 +6631,9 @@ }, "yaxes": [ { -<<<<<<< HEAD "format": "short", "label": null, "logBase": 1, -======= - "format": "s", - "label": "", - "logBase": 2, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "max": null, "min": null, "show": true @@ -9012,62 +6651,26 @@ "align": false, "alignLevel": null } - } - ], - "title": "Events", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 130, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "decimals": null, - "description": "Percentiles of Eventfeed message size", -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, "gridPos": { -<<<<<<< HEAD - "h": 7, - "w": 8, - "x": 8, - "y": 32 - }, - "hiddenSeries": false, - "id": 17, -======= "h": 8, "w": 12, "x": 0, - "y": 5 + "y": 13 }, - "id": 131, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 133, "legend": { "avg": false, -<<<<<<< HEAD - "current": true, - "max": false, - "min": false, - "rightSide": true, -======= "current": false, "max": false, "min": false, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "show": true, "total": false, "values": false @@ -9086,37 +6689,18 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}-p999", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}-p95", - "refId": "B" -======= - "expr": "sum(rate(ticdc_sorter_consume_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", + "expr": "sum(ticdc_sorter_on_disk_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}", + "legendFormat": "{{capture}}", "refId": "A" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "Event size percentile", -======= - "title": "Unified Sorter intake rate", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Unified Sorter on disk data size", "tooltip": { "shared": true, "sort": 0, @@ -9136,7 +6720,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -9154,1362 +6738,54 @@ } }, { -<<<<<<< HEAD "aliasColors": {}, - "bars": true, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of errors that interrupt Eventfeed RPC", "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 32 + "h": 8, + "w": 12, + "x": 12, + "y": 13 }, - "hiddenSeries": false, - "id": 28, + "id": 134, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(increase(ticdc_kvclient_event_feed_error_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (type)", - "format": "time_series", - "hide": false, - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "refId": "A" - }, - { - "expr": "-sum(increase(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))", - "format": "time_series", - "hide": false, - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "transfer-leader", - "refId": "B" - }, - { - "expr": "-sum(increase(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))", - "format": "time_series", - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "move-region", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Eventfeed error/m", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of KV client received events from TiKV per seconds", -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "fill": 1, - "gridPos": { -<<<<<<< HEAD - "h": 7, - "w": 8, - "x": 0, - "y": 39 - }, - "hiddenSeries": false, - "id": 29, -======= - "h": 8, - "w": 12, - "x": 12, - "y": 5 - }, - "id": 132, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "legend": { - "avg": false, -<<<<<<< HEAD - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": false, -======= - "current": false, - "max": false, - "min": false, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { -<<<<<<< HEAD - "expr": "sum(rate(ticdc_kvclient_pull_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}", -======= - "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, -<<<<<<< HEAD - "title": "KV client receive events/s", -======= - "title": "Unified Sorter event output rate", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of events that puller outputs to sorter \n per second", -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "fill": 1, - "gridPos": { -<<<<<<< HEAD - "h": 7, - "w": 8, - "x": 8, - "y": 39 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": false, -======= - "h": 8, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 133, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { -<<<<<<< HEAD - "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}", -======= - "expr": "sum(ticdc_sorter_on_disk_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, -<<<<<<< HEAD - "title": "Puller output events/s", -======= - "title": "Unified Sorter on disk data size", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { -<<<<<<< HEAD - "format": "none", -======= - "format": "bytes", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of events that are buffered in Processor's output channel and Mounter input channel", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 39 - }, - "hiddenSeries": false, - "id": 107, -======= - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 13 - }, - "id": 134, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "legend": { - "avg": false, -<<<<<<< HEAD - "current": true, - "hideEmpty": false, - "max": true, - "min": false, - "rightSide": false, -======= - "current": false, - "max": false, - "min": false, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { -<<<<<<< HEAD - "expr": "sum(ticdc_mounter_input_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}-mounter input chan", - "refId": "A" - }, - { - "expr": "-sum(ticdc_sink_buffer_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}-sink buffer chan", - "refId": "B" -======= - "expr": "sum(ticdc_sorter_in_memory_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}", - "refId": "A" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, -<<<<<<< HEAD - "title": "Sink/Mounter buffer size", -======= - "title": "Unified Sorter in-memory data size", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { -<<<<<<< HEAD - "format": "short", -======= - "format": "bytes", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cards": { -<<<<<<< HEAD - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "Duration of sorting unsorted events", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 46 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 99, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Entry sorter sort duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "Percentiles of sorting events duration", - "fill": 1, - "fillGradient": 0, -======= - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "max": null, - "min": null, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "gridPos": { - "h": 8, - "w": 12, -<<<<<<< HEAD - "x": 12, - "y": 46 - }, - "hiddenSeries": false, - "id": 53, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true -======= - "x": 0, - "y": 21 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 135, - "legend": { - "show": false ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - }, - "links": [], -<<<<<<< HEAD - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "paceLength": 10, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}-p999", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{capture}}-p95", - "refId": "B" -======= - "reverseYBuckets": false, - "targets": [ - { - "expr": "sum(rate(ticdc_sorter_flush_count_histogram_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "refId": "A" ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - } - ], - "timeFrom": null, - "timeShift": null, -<<<<<<< HEAD - "title": "Entry sorter sort duration percentile", -======= - "title": "Unified Sorter flush sizes", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "none", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, -<<<<<<< HEAD - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } -======= - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateBlues", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Duration of merging sorted events", -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "gridPos": { - "h": 8, - "w": 12, -<<<<<<< HEAD - "x": 0, - "y": 53 -======= - "x": 12, - "y": 21 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, -<<<<<<< HEAD - "id": 105, -======= - "id": 136, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { -<<<<<<< HEAD - "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", -======= - "expr": "sum(rate(ticdc_sorter_merge_count_histogram_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (le)", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "format": "heatmap", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "refId": "A" - } - ], -<<<<<<< HEAD - "title": "Entry sorter merge duration", -======= - "timeFrom": null, - "timeShift": null, - "title": "Unified Sorter merge size", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { -<<<<<<< HEAD - "decimals": 1, - "format": "s", -======= - "decimals": null, - "format": "none", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - } - ], - "title": "Unified Sorter", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 266, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Percentiles of merging sorted events duration", -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, -<<<<<<< HEAD - "x": 12, - "y": 53 - }, - "hiddenSeries": false, - "id": 106, -======= - "x": 0, - "y": 16 - }, - "id": 289, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { -<<<<<<< HEAD - "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", -======= - "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{tidb_cluster=\"$tidb_cluster\", job=\"pd\"}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "format": "time_series", - "hide": false, - "intervalFactor": 1, -<<<<<<< HEAD - "legendFormat": "{{capture}}-p999", -======= - "legendFormat": "{{instance}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{capture}}-p95", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, -<<<<<<< HEAD - "title": "Entry sorter merge duration percentile", -======= - "title": "Etcd MVCC DB total size", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { -<<<<<<< HEAD - "format": "s", -======= - "format": "decbytes", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "label": null, - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Duration of unmarshal events from kv to SQL row", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 60 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 101, -======= - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 16 - }, - "hiddenSeries": false, - "id": 114, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, -<<<<<<< HEAD -======= - "hideEmpty": true, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { -<<<<<<< HEAD - "expr": "max(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", -======= - "expr": "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p999-{{instance}}", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "refId": "A" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p99-{{instance}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p95-{{instance}}", - "refId": "C" - } - ], -<<<<<<< HEAD - "title": "Mounter unmarshal duration", -======= - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Etcd health check duration", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, -<<<<<<< HEAD - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null -======= - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": 1, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Percentiles of unmarshal events from kv to SQL row duration", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 60 - }, - "hiddenSeries": false, - "id": 55, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] -======= - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 23 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 267, - "legend": { - "show": true ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { -<<<<<<< HEAD - "expr": "histogram_quantile(0.99, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{capture}}-p99", -======= - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "queryType": "randomWalk", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "refId": "A" - }, - { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", - "format": "time_series", - "hide": true, - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{capture}}-p999", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, -<<<<<<< HEAD - "title": "Mounter unmarshal duration percentile", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" -======= - "title": "EtcdWorker exec etcd txn duration", - "tooltip": { - "show": true, - "showHistogram": true ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, -<<<<<<< HEAD - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } -======= - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of KV client dispatched event per second", -======= - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, -<<<<<<< HEAD - "x": 0, - "y": 67 - }, - "hiddenSeries": false, - "id": 31, -======= - "x": 12, - "y": 23 - }, - "hiddenSeries": false, - "id": 264, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "6.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*batch-resolved/", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { -<<<<<<< HEAD - "expr": "sum(rate(ticdc_kvclient_send_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture, changefeed, type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}-{{type}}", - "refId": "A" - }, - { - "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture, changefeed, table)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}-batch-resolved", -======= - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "interval": "", - "legendFormat": "{{capture}}-95", - "queryType": "randomWalk", - "refId": "A" - }, + "stack": false, + "steppedLine": false, + "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, - "interval": "", - "legendFormat": "{{capture}}-99", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "refId": "B" + "expr": "sum(ticdc_sorter_in_memory_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{capture}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "KV client dispatch events/s", -======= - "title": "EtcdWorker tick reactor duration", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Unified Sorter in-memory data size", "tooltip": { "shared": true, "sort": 0, @@ -10525,19 +6801,15 @@ }, "yaxes": [ { -<<<<<<< HEAD - "format": "short", -======= - "format": "s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -10550,206 +6822,53 @@ "alignLevel": null } }, -<<<<<<< HEAD -======= { "cards": { - "cardPadding": 0, - "cardRound": 0 + "cardPadding": null, + "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", + "colorScheme": "interpolateOranges", "exponent": 0.5, "max": null, - "min": 1, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 30 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 256, - "legend": { - "show": true - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "EtcdWorker exec etcd txn duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The size of batch resolved ts message from TiKV", -======= - "description": "", - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "gridPos": { - "h": 7, + "h": 8, "w": 12, - "x": 12, -<<<<<<< HEAD - "y": 67 + "x": 0, + "y": 21 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 97, -======= - "y": 30 - }, - "hiddenSeries": false, - "id": 258, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 135, "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true + "show": false }, -<<<<<<< HEAD "links": [], "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", + "expr": "sum(rate(ticdc_sorter_flush_count_histogram_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (le)", "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", -======= - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "6.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{capture}}-p95", - "queryType": "randomWalk", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{le}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{capture}}-p99", - "refId": "B" } ], -<<<<<<< HEAD - "title": "KV client batch resolved size", -======= - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "EtcdWorker exec etcd txn duration percentile", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Unified Sorter flush sizes", "tooltip": { "show": true, - "showHistogram": true + "showHistogram": false }, - "tooltipDecimals": 1, "type": "heatmap", "xAxis": { "show": true @@ -10757,7 +6876,7 @@ "xBucketNumber": null, "xBucketSize": null, "yAxis": { - "decimals": 1, + "decimals": null, "format": "none", "logBase": 1, "max": null, @@ -10765,34 +6884,9 @@ "show": true, "splitFactor": null }, -<<<<<<< HEAD - "yBucketBound": "upper", + "yBucketBound": "auto", "yBucketNumber": null, "yBucketSize": null -======= - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) }, { "cards": { @@ -10802,52 +6896,43 @@ "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", - "colorScheme": "interpolatePurples", + "colorScheme": "interpolateBlues", "exponent": 0.5, - "min": 0, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, "gridPos": { - "h": 7, + "h": 8, "w": 12, - "x": 0, - "y": 37 + "x": 12, + "y": 21 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 254, + "id": 136, "legend": { - "show": true + "show": false }, - "pluginVersion": "6.1.6", + "links": [], "reverseYBuckets": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", + "expr": "sum(rate(ticdc_sorter_merge_count_histogram_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (le)", "format": "heatmap", - "instant": false, - "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", - "queryType": "randomWalk", "refId": "A" } ], - "title": "EtcdWorker txn size", + "timeFrom": null, + "timeShift": null, + "title": "Unified Sorter merge size", "tooltip": { "show": true, - "showHistogram": true + "showHistogram": false }, - "tooltipDecimals": null, "type": "heatmap", "xAxis": { "show": true @@ -10856,7 +6941,7 @@ "xBucketSize": null, "yAxis": { "decimals": null, - "format": "decbytes", + "format": "none", "logBase": 1, "max": null, "min": null, @@ -10866,40 +6951,35 @@ "yBucketBound": "auto", "yBucketNumber": null, "yBucketSize": null - }, + } + ], + "title": "Unified Sorter", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 266, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of regions that are being scanned", -======= - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, -<<<<<<< HEAD "x": 0, - "y": 74 - }, - "hiddenSeries": false, - "id": 177, -======= - "x": 12, - "y": 37 + "y": 16 }, - "hiddenSeries": false, - "id": 260, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 289, "legend": { "alignAsTable": true, "avg": false, @@ -10915,18 +6995,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", - "options": { -<<<<<<< HEAD - "dataLinks": [] - }, - "paceLength": 10, -======= - "alertThreshold": true - }, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "percentage": false, - "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", @@ -10936,38 +7007,19 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(ticdc_kvclient_region_token{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", + "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{tidb_cluster=\"$tidb_cluster\", job=\"pd\"}", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", -======= - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "interval": "", - "legendFormat": "{{capture}}-p95", - "queryType": "randomWalk", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{instance}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", - "hide": false, - "interval": "", - "legendFormat": "{{capture}}-p99", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "KV client scanning regions", -======= - "title": "EtcdWorker txn size percentile", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Etcd MVCC DB total size", "tooltip": { "shared": true, "sort": 0, @@ -10983,7 +7035,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "decbytes", "label": null, "logBase": 1, "max": null, @@ -10996,7 +7048,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -11010,43 +7062,25 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "Active stream count of each gRPC connection", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 74 + "y": 16 }, "hiddenSeries": false, - "id": 188, -======= - "description": "The time consumed of writing WAL into the persistent storage in .99", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 44 - }, - "id": 291, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "id": 114, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, - "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 300, "total": false, "values": true }, @@ -11054,15 +7088,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", -<<<<<<< HEAD "options": { "dataLinks": [] }, -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "paceLength": 10, "percentage": false, - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -11071,37 +7102,36 @@ "steppedLine": false, "targets": [ { -<<<<<<< HEAD - "expr": "sum(ticdc_kvclient_grpc_stream_count{tidb_cluster=\"$tidb_cluster\"}) by (store)", + "expr": "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{store}}", + "legendFormat": "p999-{{instance}}", "refId": "A" -======= - "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, le))", + }, + { + "expr": "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", - "refId": "A", - "step": 4 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "intervalFactor": 1, + "legendFormat": "p99-{{instance}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p95-{{instance}}", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "KV client gRPC stream count", -======= - "title": "Etcd 99% WAL fsync duration", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "Etcd health check duration", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -11113,11 +7143,7 @@ }, "yaxes": [ { -<<<<<<< HEAD - "format": "short", -======= "format": "s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "label": null, "logBase": 1, "max": null, @@ -11139,149 +7165,80 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 1, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", -<<<<<<< HEAD - "description": "The number of regions that have not connected to TiKV", + "description": "", "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 81 - }, - "hiddenSeries": false, - "id": 251, -======= - "description": "The time consumed of handling etcd transactions in .99", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 44 + "y": 23 }, - "id": 290, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 267, "legend": { - "alignAsTable": true, - "avg": false, - "current": true, -<<<<<<< HEAD - "hideEmpty": true, - "hideZero": true, -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 300, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", -<<<<<<< HEAD - "options": { - "alertThreshold": true + "show": true }, -======= ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, "targets": [ { -<<<<<<< HEAD "exemplar": true, - "expr": "sum(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", - "format": "time_series", + "expr": "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", - "refId": "A" -======= - "expr": "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, result, le))", - "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} {{result}}", - "refId": "A", - "step": 4 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, -<<<<<<< HEAD - "title": "KV client cached regions", -======= - "title": "Etcd 99% Handle transactions duration", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "title": "EtcdWorker exec etcd txn duration", "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" + "show": true, + "showHistogram": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, - "yaxes": [ - { -<<<<<<< HEAD - "format": "short", -======= - "format": "s", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } -<<<<<<< HEAD + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null }, { "aliasColors": {}, @@ -11289,11 +7246,9 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Estimate the remaining time for a changefeed initialization (on a specific capture)", + "description": "", "fieldConfig": { - "defaults": { - "unit": "s" - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -11302,32 +7257,27 @@ "h": 7, "w": 12, "x": 12, - "y": 81 + "y": 23 }, "hiddenSeries": false, - "id": 252, + "id": 264, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, + "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", @@ -11338,19 +7288,26 @@ "targets": [ { "exemplar": true, - "expr": "abs(sum(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"} / deriv(ticdc_kvclient_cached_region{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture, changefeed, store))", - "format": "time_series", + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", "interval": "", - "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", + "legendFormat": "{{capture}}-95", + "queryType": "randomWalk", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "hide": false, + "interval": "", + "legendFormat": "{{capture}}-99", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Estimate remaining time for initialization", + "title": "EtcdWorker tick reactor duration", "tooltip": { "shared": true, "sort": 0, @@ -11367,14 +7324,14 @@ "yaxes": [ { "format": "s", - "label": "", - "logBase": 2, + "label": null, + "logBase": 1, "max": null, "min": null, "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -11386,40 +7343,106 @@ "align": false, "alignLevel": null } - } - ], - "title": "Events", -======= - } - ], - "title": "Etcd", ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 130, - "panels": [ + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 1, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 30 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 256, + "legend": { + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "EtcdWorker exec etcd txn duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 0, - "y": 7 + "x": 12, + "y": 30 }, - "id": 131, + "hiddenSeries": false, + "id": 258, "legend": { "avg": false, "current": false, @@ -11431,9 +7454,12 @@ }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", @@ -11443,18 +7469,31 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_sorter_consume_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}", + "legendFormat": "{{capture}}-p95", + "queryType": "randomWalk", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{capture}}-p99", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Unified Sorter intake rate", + "title": "EtcdWorker exec etcd txn duration percentile", "tooltip": { "shared": true, "sort": 0, @@ -11470,9 +7509,9 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -11491,20 +7530,100 @@ "alignLevel": null } }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolatePurples", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 37 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 254, + "legend": { + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "EtcdWorker txn size", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "decbytes", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, - "y": 7 + "y": 37 }, - "id": 132, + "hiddenSeries": false, + "id": 260, "legend": { "avg": false, "current": false, @@ -11516,9 +7635,12 @@ }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "6.1.6", "pointradius": 2, "points": false, "renderer": "flot", @@ -11528,18 +7650,27 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}-{{changefeed}}", + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "interval": "", + "legendFormat": "{{capture}}-p95", + "queryType": "randomWalk", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,capture))", + "hide": false, + "interval": "", + "legendFormat": "{{capture}}-p99", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Unified Sorter event output rate", + "title": "EtcdWorker txn size percentile", "tooltip": { "shared": true, "sort": 0, @@ -11555,7 +7686,7 @@ }, "yaxes": [ { - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -11582,33 +7713,39 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The time consumed of writing WAL into the persistent storage in .99", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 0, -<<<<<<< HEAD - "y": 13 -======= - "y": 14 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 44 }, - "id": 133, + "id": 291, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, "min": false, + "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -11617,22 +7754,25 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_sorter_on_disk_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, le))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Unified Sorter on disk data size", + "title": "Etcd 99% WAL fsync duration", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -11644,11 +7784,11 @@ }, "yaxes": [ { - "format": "bytes", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -11671,33 +7811,37 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The time consumed of handling etcd transactions in .99", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, "x": 12, -<<<<<<< HEAD - "y": 13 -======= - "y": 14 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 44 }, - "id": 134, + "id": 290, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -11706,186 +7850,58 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_sorter_in_memory_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", + "expr": "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, result, le))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{capture}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Unified Sorter in-memory data size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "max": null, - "min": null, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 21 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 135, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "expr": "sum(rate(ticdc_sorter_flush_count_histogram_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Unified Sorter flush sizes", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "none", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateBlues", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 21 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 136, - "legend": { - "show": false - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "expr": "sum(rate(ticdc_sorter_merge_count_histogram_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (le)", - "format": "heatmap", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{instance}} {{result}}", + "refId": "A", + "step": 4 } ], + "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Unified Sorter merge size", + "title": "Etcd 99% Handle transactions duration", "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "none", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], - "title": "Unified Sorter", + "title": "Etcd", "type": "row" }, { @@ -11894,7 +7910,7 @@ "h": 1, "w": 24, "x": 0, - "y": 5 + "y": 6 }, "id": 58, "panels": [ @@ -11911,11 +7927,7 @@ "h": 7, "w": 12, "x": 0, -<<<<<<< HEAD - "y": 5 -======= - "y": 21 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 7 }, "id": 60, "legend": { @@ -12039,15 +8051,9 @@ "grid": {}, "gridPos": { "h": 7, -<<<<<<< HEAD "w": 12, "x": 12, - "y": 5 -======= - "w": 5, - "x": 7, - "y": 21 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 7 }, "id": 74, "legend": { @@ -12154,13 +8160,8 @@ "gridPos": { "h": 7, "w": 12, -<<<<<<< HEAD "x": 0, - "y": 12 -======= - "x": 12, - "y": 21 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 14 }, "id": 147, "legend": { @@ -12244,88 +8245,6 @@ } }, { -<<<<<<< HEAD -======= - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The time consumed to CDC incremental scan", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 28 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 68, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "maxPerRow": 3, - "repeat": null, - "repeatDirection": "h", - "reverseYBuckets": false, - "targets": [ - { - "expr": "sum(rate(tikv_cdc_scan_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Initial scan duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, - { ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "aliasColors": {}, "bars": false, "dashLength": 10, @@ -12341,11 +8260,7 @@ "h": 7, "w": 12, "x": 12, -<<<<<<< HEAD - "y": 12 -======= - "y": 28 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 14 }, "id": 194, "legend": { @@ -12507,15 +8422,9 @@ "grid": {}, "gridPos": { "h": 7, -<<<<<<< HEAD "w": 7, "x": 0, - "y": 19 -======= - "w": 6, - "x": 18, - "y": 28 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 21 }, "id": 152, "legend": { @@ -12644,15 +8553,9 @@ "grid": {}, "gridPos": { "h": 7, -<<<<<<< HEAD "w": 5, "x": 7, - "y": 19 -======= - "w": 12, - "x": 0, - "y": 35 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 21 }, "id": 153, "legend": { @@ -12748,11 +8651,7 @@ "h": 7, "w": 12, "x": 12, -<<<<<<< HEAD - "y": 19 -======= - "y": 35 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 21 }, "id": 70, "legend": { @@ -12854,7 +8753,7 @@ "h": 7, "w": 12, "x": 0, - "y": 26 + "y": 28 }, "heatmap": {}, "hideZeroBuckets": true, @@ -12925,13 +8824,8 @@ "gridPos": { "h": 7, "w": 6, -<<<<<<< HEAD "x": 12, - "y": 26 -======= - "x": 18, - "y": 35 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 28 }, "id": 72, "legend": { @@ -13024,15 +8918,9 @@ "fill": 1, "gridPos": { "h": 7, -<<<<<<< HEAD "w": 6, "x": 18, - "y": 26 -======= - "w": 12, - "x": 0, - "y": 42 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 28 }, "id": 140, "legend": { @@ -13143,13 +9031,8 @@ "gridPos": { "h": 7, "w": 12, -<<<<<<< HEAD "x": 0, - "y": 33 -======= - "x": 12, - "y": 42 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 35 }, "id": 78, "legend": { @@ -13255,15 +9138,9 @@ "grid": {}, "gridPos": { "h": 7, -<<<<<<< HEAD "w": 6, "x": 12, - "y": 33 -======= - "w": 12, - "x": 0, - "y": 49 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 35 }, "id": 76, "legend": { @@ -13363,13 +9240,8 @@ "gridPos": { "h": 7, "w": 6, -<<<<<<< HEAD "x": 18, - "y": 33 -======= - "x": 12, - "y": 49 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 35 }, "id": 139, "legend": { @@ -13465,15 +9337,9 @@ "fill": 4, "gridPos": { "h": 7, -<<<<<<< HEAD "w": 12, "x": 0, - "y": 40 -======= - "w": 6, - "x": 18, - "y": 49 ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) + "y": 42 }, "id": 143, "legend": { @@ -13600,7 +9466,7 @@ "h": 7, "w": 12, "x": 12, - "y": 40 + "y": 42 }, "id": 145, "legend": { @@ -13628,23 +9494,6 @@ "pointradius": 1, "points": false, "renderer": "flot", -<<<<<<< HEAD -======= - "repeat": null, - "repeatDirection": "h", - "scopedVars": { - "instance": { - "selected": false, - "text": "172.16.5.33:47912", - "value": "172.16.5.33:47912" - }, - "runtime_instance": { - "selected": false, - "text": "172.16.5.37:47912", - "value": "172.16.5.37:47912" - } - }, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "seriesOverrides": [ { "alias": "/.*len/", @@ -13738,7 +9587,7 @@ "h": 7, "w": 12, "x": 0, - "y": 47 + "y": 49 }, "id": 141, "legend": { @@ -13766,23 +9615,6 @@ "pointradius": 5, "points": false, "renderer": "flot", -<<<<<<< HEAD -======= - "repeat": null, - "repeatDirection": "h", - "scopedVars": { - "instance": { - "selected": false, - "text": "172.16.5.33:47912", - "value": "172.16.5.33:47912" - }, - "runtime_instance": { - "selected": false, - "text": "172.16.5.37:47912", - "value": "172.16.5.37:47912" - } - }, ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -13860,7 +9692,7 @@ "h": 7, "w": 6, "x": 12, - "y": 47 + "y": 49 }, "heatmap": {}, "hideZeroBuckets": true, @@ -13931,7 +9763,7 @@ "h": 7, "w": 6, "x": 18, - "y": 47 + "y": 49 }, "id": 142, "legend": { @@ -13959,25 +9791,7 @@ "pointradius": 5, "points": false, "renderer": "flot", -<<<<<<< HEAD "seriesOverrides": [], -======= - "scopedVars": { - "instance": { - "selected": false, - "text": "172.16.5.33:47912", - "value": "172.16.5.33:47912" - }, - "runtime_instance": { - "selected": false, - "text": "172.16.5.37:47912", - "value": "172.16.5.37:47912" - } - }, - "seriesOverrides": [ - {} - ], ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) "spaceLength": 10, "stack": false, "steppedLine": false, @@ -14055,7 +9869,7 @@ "h": 1, "w": 24, "x": 0, - "y": 6 + "y": 7 }, "id": 155, "panels": [ @@ -14298,8 +10112,8 @@ }, "runtime_instance": { "selected": false, - "text": "172.16.5.32:47800", - "value": "172.16.5.32:47800" + "text": "172.16.5.37:47912", + "value": "172.16.5.37:47912" } }, "seriesOverrides": [], @@ -14405,8 +10219,8 @@ }, "runtime_instance": { "selected": false, - "text": "172.16.5.32:47800", - "value": "172.16.5.32:47800" + "text": "172.16.5.37:47912", + "value": "172.16.5.37:47912" } }, "seriesOverrides": [ @@ -14527,8 +10341,8 @@ }, "runtime_instance": { "selected": false, - "text": "172.16.5.32:47800", - "value": "172.16.5.32:47800" + "text": "172.16.5.37:47912", + "value": "172.16.5.37:47912" } }, "seriesOverrides": [ @@ -14829,11 +10643,5 @@ "timezone": "browser", "title": "Test-Cluster-TiCDC", "uid": "YiGL8hBZ1", -<<<<<<< HEAD - "version": 25 -} - -======= "version": 29 -} ->>>>>>> 20e3f139f (metrics(ticdc): add resolved ts and add changefeed to dataflow (#4038)) +} \ No newline at end of file