From 0bd8d2e1600a0d9dd00ecf8a3f2e00d3d143e19c Mon Sep 17 00:00:00 2001 From: BigMoney Zheng <11624213+buggithubs@users.noreply.github.com> Date: Thu, 4 Feb 2021 16:42:03 +0800 Subject: [PATCH 1/2] binlog json add TSOmore info & alert rule --- metrics/grafana/binlog.json | 1331 +++++++++++++++++++++++++++++++++-- 1 file changed, 1258 insertions(+), 73 deletions(-) diff --git a/metrics/grafana/binlog.json b/metrics/grafana/binlog.json index 1adb4a96a..ded83471e 100644 --- a/metrics/grafana/binlog.json +++ b/metrics/grafana/binlog.json @@ -1,8 +1,8 @@ { "__inputs": [ { - "name": "DS_TEST-CLUSTER", - "label": "test-cluster", + "name": "DS_TC1", + "label": "binlog", "description": "", "type": "datasource", "pluginId": "prometheus", @@ -10,6 +10,12 @@ } ], "__requires": [ + { + "type": "panel", + "id": "alertlist", + "name": "Alert List", + "version": "" + }, { "type": "grafana", "id": "grafana", @@ -33,17 +39,31 @@ "id": "singlestat", "name": "Singlestat", "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" } ], "annotations": { "list": [ { "builtIn": 1, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "pluginId": "prometheus", + "pluginName": "Prometheus", "type": "dashboard" } ] @@ -52,17 +72,773 @@ "gnetId": null, "graphTooltip": 0, "id": null, - "iteration": 1569404109122, + "iteration": 1612171134802, "links": [], "panels": [ { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 80, + "panels": [], + "title": "TSO - pipline", + "type": "row" + }, + { + "content": "\n# TSO\n\n1. PUMP GC TSO < PUMP max commit TSO \n2. PUMP GC TSO < Drainer Pull PUMP handle TSO\n3. PUMP GC TSO < Draienr checkpoint TSO\n\n## alert\n\n1. drainer checkpoint tso no change for 1m\n2. drainer checkpoint tso delay > 1 hour\n3. PUMP disk space <= 10G\n\n\n### read more: https://docs.pingcap.com/zh/tidb/stable/tidb-binlog-overview\n\n", + "datasource": "${DS_TC1}", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 93, + "links": [], + "mode": "markdown", + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Tips ", + "type": "text" + }, + { + "dashboardFilter": "", + "dashboardTags": [], + "datasource": "${DS_TC1}", + "folderId": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 89, + "limit": 10, + "links": [], + "nameFilter": "", + "onlyAlertsOnDashboard": true, + "show": "current", + "sortOrder": 1, + "stateFilter": [ + "alerting", + "pending", + "no_data", + "execution_error" + ], + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Binlog alert", + "type": "alertlist" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TC1}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 85, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "binlog_pump_storage_gc_ts", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} : gc_tso", + "refId": "A" + }, + { + "expr": "binlog_pump_storage_max_commit_ts", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{instance}} : max_commit_tso", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Binlog File GC TSO --- Metadata", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dateTimeAsIso", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TC1}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 84, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "binlog_pump_storage_gc_ts", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{instance}} : gc_tso", + "refId": "A" + }, + { + "expr": "binlog_pump_storage_max_commit_ts", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} : max_commit_tso", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "New Binlog File TSO -- Metadata", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dateTimeAsIso", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TC1}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 90, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "binlog_drainer_pump_position{instance = \"$drainer_instance\"}", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{nodeID}}", + "refId": "A" + }, + { + "expr": "irate(binlog_drainer_checkpoint_tso[1m])", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "B" + }, + { + "expr": "(time() - binlog_drainer_checkpoint_tso / 1000)", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Drainer Pull Pump Handle TSO", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dateTimeAsIso", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "${DS_TC1}", + "fontSize": "130%", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 15 + }, + "hideTimeOverride": false, + "id": 87, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "styles": [ + { + "alias": "Drainer instance", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Metric", + "type": "string" + }, + { + "alias": "Drainer checkpoint TSO", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Current", + "thresholds": [], + "type": "date", + "unit": "short" + } + ], + "targets": [ + { + "expr": "binlog_drainer_checkpoint_tso", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Drainer checkpoint TSO", + "transform": "timeseries_aggregations", + "type": "table" + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 500 + ], + "type": "lt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "Drainer checkpoit no cheange for 5m", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TC1}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 86, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "binlog_drainer_pump_position{instance = \"$drainer_instance\"}", + "format": "time_series", + "hide": true, + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{nodeID}}", + "refId": "A" + }, + { + "expr": "irate(binlog_drainer_checkpoint_tso[1m]) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} change[1m]", + "refId": "B" + }, + { + "expr": "(time() - binlog_drainer_checkpoint_tso / 1000)", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{instance}} delay time", + "refId": "C" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 500 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Drainer checkpoit no cheange for 5m", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 3600 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "C", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "Drainer checkpoint delay > 1 hour", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TC1}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 91, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*change.*/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "binlog_drainer_pump_position{instance = \"$drainer_instance\"}", + "format": "time_series", + "hide": true, + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{nodeID}}", + "refId": "A" + }, + { + "expr": "irate(binlog_drainer_checkpoint_tso[1m]) ", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} change[1m]", + "refId": "B" + }, + { + "expr": "(time() - binlog_drainer_checkpoint_tso / 1000)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}} delay time", + "refId": "C" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 3600 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Drainer checkpoint delay > 1 hour", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, "id": 73, "panels": [ { @@ -70,13 +846,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 1 + "y": 30 }, "hideTimeOverride": false, "id": 68, @@ -102,11 +878,11 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "binlog_pump_storage_storage_size_bytes", + "expr": "binlog_pump_storage_storage_size_bytes{type=\"capacity\"}", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -118,7 +894,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Storage Size", + "title": "capacity -- Storage Size", "tooltip": { "shared": true, "sort": 0, @@ -156,17 +932,249 @@ } }, { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 107374182400 + ], + "type": "lt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "1m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "message": "PUMP The disk space is less than 10G", + "name": "PUMP disk space <= 10G", + "noDataState": "no_data", + "notifications": [] + }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", + "description": "", "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 1 + "y": 30 + }, + "hideTimeOverride": false, + "id": 81, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideZero": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "binlog_pump_storage_storage_size_bytes{type=\"available\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}} : {{type}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 107374182400 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "available -- Storage Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TC1}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 83, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "binlog_pump_storage_gc_ts", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} : gc_tso", + "refId": "A" + }, + { + "expr": "binlog_pump_storage_max_commit_ts", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{instance}} : max_commit_tso", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Binlog File GC TSO --- Metadata", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dateTimeAsIso", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TC1}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 37 }, "id": 63, "legend": { @@ -198,6 +1206,7 @@ { "expr": "binlog_pump_storage_gc_ts", "format": "time_series", + "hide": true, "intervalFactor": 2, "legendFormat": "{{instance}} : gc_tso", "refId": "A" @@ -214,7 +1223,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Metadata", + "title": "New Binlog File TSO -- Metadata", "tooltip": { "shared": true, "sort": 0, @@ -256,7 +1265,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -265,7 +1274,7 @@ "h": 7, "w": 12, "x": 0, - "y": 8 + "y": 44 }, "id": 7, "legend": { @@ -276,6 +1285,8 @@ "min": false, "rightSide": true, "show": true, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, @@ -345,11 +1356,45 @@ } }, { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 1 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "PUMP Write Binlog Latency alert", + "noDataState": "no_data", + "notifications": [] + }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -358,7 +1403,7 @@ "h": 7, "w": 12, "x": 12, - "y": 8 + "y": 44 }, "id": 3, "legend": { @@ -400,7 +1445,15 @@ "refId": "C" } ], - "thresholds": [], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1 + } + ], "timeFrom": null, "timeRegions": [], "timeShift": null, @@ -447,13 +1500,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 15 + "y": 51 }, "id": 44, "legend": { @@ -541,7 +1594,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -550,7 +1603,7 @@ "h": 7, "w": 12, "x": 12, - "y": 15 + "y": 51 }, "id": 66, "legend": { @@ -639,13 +1692,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 22 + "y": 58 }, "id": 48, "legend": { @@ -727,13 +1780,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 22 + "y": 58 }, "id": 67, "legend": { @@ -814,13 +1867,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 29 + "y": 65 }, "id": 76, "legend": { @@ -907,7 +1960,7 @@ "h": 1, "w": 24, "x": 0, - "y": 1 + "y": 30 }, "id": 74, "panels": [ @@ -920,7 +1973,7 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "format": "dateTimeAsIso", "gauge": { "maxValue": null, @@ -933,7 +1986,7 @@ "h": 7, "w": 8, "x": 0, - "y": 23 + "y": 17 }, "hideTimeOverride": false, "id": 70, @@ -986,7 +2039,6 @@ "timeFrom": null, "timeShift": null, "title": "Checkpoint TSO", - "transparent": false, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -1003,13 +2055,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "fill": 1, "gridPos": { "h": 7, "w": 16, "x": 8, - "y": 23 + "y": 17 }, "id": 69, "legend": { @@ -1050,6 +2102,7 @@ ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, "title": "Pump Handle TSO", "tooltip": { @@ -1057,7 +2110,6 @@ "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "buckets": null, @@ -1083,14 +2135,18 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -1099,7 +2155,7 @@ "h": 7, "w": 12, "x": 0, - "y": 30 + "y": 24 }, "id": 62, "legend": { @@ -1138,6 +2194,7 @@ ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, "title": "Pull Binlog QPS by Pump NodeID", "tooltip": { @@ -1171,14 +2228,18 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -1187,7 +2248,7 @@ "h": 7, "w": 12, "x": 12, - "y": 30 + "y": 24 }, "id": 53, "legend": { @@ -1226,6 +2287,7 @@ ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, "title": "95% Binlog Reach Duration By Pump", "tooltip": { @@ -1259,14 +2321,18 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -1275,7 +2341,7 @@ "h": 7, "w": 12, "x": 0, - "y": 37 + "y": 31 }, "id": 58, "legend": { @@ -1314,6 +2380,7 @@ ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, "title": "Error By Type", "tooltip": { @@ -1347,14 +2414,18 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -1363,7 +2434,7 @@ "h": 7, "w": 12, "x": 12, - "y": 37 + "y": 31 }, "id": 6, "legend": { @@ -1402,6 +2473,7 @@ ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, "title": "Drainer Event", "tooltip": { @@ -1435,14 +2507,52 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 1 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "Drainer to downstream exec SQL 99%", + "noDataState": "no_data", + "notifications": [] + }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -1451,7 +2561,7 @@ "h": 7, "w": 12, "x": 0, - "y": 44 + "y": 38 }, "id": 15, "legend": { @@ -1486,8 +2596,17 @@ "step": 2 } ], - "thresholds": [], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1 + } + ], "timeFrom": null, + "timeRegions": [], "timeShift": null, "title": "99% Execute Time", "tooltip": { @@ -1521,14 +2640,52 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 1 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "1m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "Drainers 99% SQL query", + "noDataState": "no_data", + "notifications": [] + }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -1537,7 +2694,7 @@ "h": 7, "w": 12, "x": 12, - "y": 44 + "y": 38 }, "id": 71, "legend": { @@ -1572,8 +2729,17 @@ "step": 2 } ], - "thresholds": [], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1 + } + ], "timeFrom": null, + "timeRegions": [], "timeShift": null, "title": "99% sql query Time", "tooltip": { @@ -1607,14 +2773,18 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -1623,7 +2793,7 @@ "h": 7, "w": 12, "x": 0, - "y": 51 + "y": 45 }, "id": 55, "legend": { @@ -1662,6 +2832,7 @@ ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, "title": "95% Binlog Size", "tooltip": { @@ -1695,14 +2866,18 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -1711,7 +2886,7 @@ "h": 7, "w": 12, "x": 12, - "y": 51 + "y": 45 }, "id": 52, "legend": { @@ -1751,6 +2926,7 @@ ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, "title": "DDL Job Count", "tooltip": { @@ -1784,20 +2960,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "fill": 1, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 58 + "y": 52 }, "id": 72, "legend": { @@ -1832,6 +3012,7 @@ ], "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, "title": "queue size", "tooltip": { @@ -1864,7 +3045,11 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "repeat": null, @@ -1877,7 +3062,7 @@ "h": 1, "w": 24, "x": 0, - "y": 2 + "y": 31 }, "id": 75, "panels": [ @@ -1886,7 +3071,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -1974,7 +3159,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "editable": true, "error": false, "fill": 1, @@ -2072,7 +3257,7 @@ { "allValue": null, "current": {}, - "datasource": "${DS_TEST-CLUSTER}", + "datasource": "${DS_TC1}", "definition": "", "hide": 0, "includeAll": false, @@ -2123,7 +3308,7 @@ ] }, "timezone": "browser", - "title": "Test-Cluster-Binlog", - "uid": "RDdDTFvZz", - "version": 9 -} \ No newline at end of file + "title": "Binlog", + "uid": "RDdDTFvZz4", + "version": 20 +} From 9a70bf3b52e48a2619811dab76df1d8a62612039 Mon Sep 17 00:00:00 2001 From: BigMoney Zheng <11624213+buggithubs@users.noreply.github.com> Date: Thu, 4 Feb 2021 16:42:36 +0800 Subject: [PATCH 2/2] binlog json add TSO more info & alert rule