INF-221 Further testing of Grafana Alerts on Clean Dev Environments (#…

…3715)
AudiusProject · Aug 18, 2022 · 7bdb640 · 7bdb640
1 parent 862e0af
commit 7bdb640
Show file tree

Hide file tree

Showing 18 changed files with 210 additions and 317 deletions.
diff --git a/monitoring/README.md b/monitoring/README.md
@@ -5,6 +5,10 @@
 ```bash
 A run monitoring up
 
+# on first start, wait a few minutes, then upload the dashboards again
+# this will fix a recent regression with missing Library panels
+# ./grafana/bin/upload-dashboards.sh
+
 # A run monitoring down
 
 # optionally remove all grafana and prometheus data

diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml
@@ -24,7 +24,6 @@ services:
     build:
       context: grafana
       args:
-        - GRAFANA_VERSION=latest
         - GF_INSTALL_IMAGE_RENDERER_PLUGIN=true
     user: 0:0
     env_file:

diff --git a/monitoring/grafana/Dockerfile b/monitoring/grafana/Dockerfile
@@ -1,7 +1,7 @@
 # Docs: https://grafana.com/docs/grafana/latest/installation/docker/#build-and-run-a-docker-image-with-pre-installed-plugins
 # Original: https://github.com/grafana/grafana/blob/a51c2774b8e77cafc0100a3882d77039c189e301/packaging/docker/custom/ubuntu.Dockerfile 
 
-ARG GRAFANA_VERSION="latest"
+ARG GRAFANA_VERSION="9.1.0"
 
 FROM grafana/grafana:${GRAFANA_VERSION}
 

diff --git a/monitoring/grafana/alerts/alert.template.json b/monitoring/grafana/alerts/alert.template.json
@@ -2,14 +2,14 @@
   "id": {alert_id},
   "uid": "{alert_uid}",
   "orgID": 1,
-  "folderUID": "ARwYXYzVk",
+  "folderUID": "zpejR_iVz",
   "ruleGroup": "group",
   "title": "{title}",
   "condition": "{condition_ref}",
   "data": {data},
   "noDataState": "OK",
   "execErrState": "Alerting",
-  "for": "0",
+  "for": "0s",
   "annotations": {{
     "__alertId__": "{alert_uid}",
     "__dashboardUid__": "{dashboard_uid}",

diff --git a/monitoring/grafana/alerts/audius-governance_new-proposals.json b/monitoring/grafana/alerts/audius-governance_new-proposals.json
@@ -1,7 +1,7 @@
 [
   {
     "annotations": {
-      "__alertId__": "YCgu57m4z_015_Notification",
+      "__alertId__": "YCgu57m4z_015_High",
       "__dashboardUid__": "YCgu57m4z",
       "__panelId__": "15",
       "description": "\n\nAny more than 2 should trigger multiple alarms (including this one) since an attacker may be self-signing multiple proposals.",
@@ -37,7 +37,7 @@
             {
               "evaluator": {
                 "params": [
-                  null,
+                  2,
                   0
                 ],
                 "type": "gt"
@@ -77,16 +77,16 @@
       }
     ],
     "execErrState": "Alerting",
-    "folderUID": "ARwYXYzVk",
-    "for": "0",
-    "id": 220154,
+    "folderUID": "zpejR_iVz",
+    "for": "0s",
+    "id": 180151,
     "labels": {
-      "channel": "notification"
+      "channel": "high-alert"
     },
     "noDataState": "OK",
     "orgID": 1,
     "ruleGroup": "group",
-    "title": "New Proposals (Notification)",
-    "uid": "YCgu57m4z_015_Notification"
+    "title": "New Proposals (High)",
+    "uid": "YCgu57m4z_015_High"
   }
 ]
diff --git a/monitoring/grafana/alerts/audius-governance_open-proposals.json b/monitoring/grafana/alerts/audius-governance_open-proposals.json
@@ -1,7 +1,7 @@
 [
   {
     "annotations": {
-      "__alertId__": "YCgu57m4z_020_Notification",
+      "__alertId__": "YCgu57m4z_020_Low",
       "__dashboardUid__": "YCgu57m4z",
       "__panelId__": "20",
       "description": "\n\nThe number open proposals should typically be 1, which will send a Slack notification.\n\n2 opened PRs at the same time will aggressively escalate to on-call.",
@@ -37,7 +37,7 @@
             {
               "evaluator": {
                 "params": [
-                  null,
+                  1,
                   0
                 ],
                 "type": "gt"
@@ -77,16 +77,106 @@
       }
     ],
     "execErrState": "Alerting",
-    "folderUID": "ARwYXYzVk",
-    "for": "0",
-    "id": 220204,
+    "folderUID": "zpejR_iVz",
+    "for": "0s",
+    "id": 180203,
     "labels": {
-      "channel": "notification"
+      "channel": "low-alert"
     },
     "noDataState": "OK",
     "orgID": 1,
     "ruleGroup": "group",
-    "title": "Open Proposals (Notification)",
-    "uid": "YCgu57m4z_020_Notification"
+    "title": "Open Proposals (Low)",
+    "uid": "YCgu57m4z_020_Low"
+  },
+  {
+    "annotations": {
+      "__alertId__": "YCgu57m4z_020_High",
+      "__dashboardUid__": "YCgu57m4z",
+      "__panelId__": "20",
+      "description": "\n\nThe number open proposals should typically be 1, which will send a Slack notification.\n\n2 opened PRs at the same time will aggressively escalate to on-call.",
+      "runbook_url": "http://grafana.audius.co/d/YCgu57m4z?viewPanel=20",
+      "summary": "The number of open proposals at the moment.\n\n"
+    },
+    "condition": "B",
+    "data": [
+      {
+        "datasourceUid": "r2_nnDL7z",
+        "model": {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "r2_nnDL7z"
+          },
+          "expr": "audius_exporters_sdk_proposals{outcome=\"InProgress\"}",
+          "hide": false,
+          "intervalMs": 1000,
+          "maxDataPoints": 43200,
+          "refId": "A"
+        },
+        "queryType": "",
+        "refId": "A",
+        "relativeTimeRange": {
+          "from": 600,
+          "to": 0
+        }
+      },
+      {
+        "datasourceUid": "-100",
+        "model": {
+          "conditions": [
+            {
+              "evaluator": {
+                "params": [
+                  2,
+                  0
+                ],
+                "type": "gt"
+              },
+              "operator": {
+                "type": "or"
+              },
+              "query": {
+                "params": [
+                  "A"
+                ]
+              },
+              "reducer": {
+                "params": [],
+                "type": "last"
+              },
+              "type": "query"
+            }
+          ],
+          "datasource": {
+            "name": "Expression",
+            "type": "__expr__",
+            "uid": "__expr__"
+          },
+          "hide": false,
+          "intervalMs": 1000,
+          "maxDataPoints": 43200,
+          "refId": "B",
+          "type": "classic_conditions"
+        },
+        "queryType": "",
+        "refId": "B",
+        "relativeTimeRange": {
+          "from": 0,
+          "to": 0
+        }
+      }
+    ],
+    "execErrState": "Alerting",
+    "folderUID": "zpejR_iVz",
+    "for": "0s",
+    "id": 180201,
+    "labels": {
+      "channel": "high-alert"
+    },
+    "noDataState": "OK",
+    "orgID": 1,
+    "ruleGroup": "group",
+    "title": "Open Proposals (High)",
+    "uid": "YCgu57m4z_020_High"
   }
 ]
diff --git a/monitoring/grafana/alerts/audius-governance_proposals-by-unknown-proposers.json b/monitoring/grafana/alerts/audius-governance_proposals-by-unknown-proposers.json
@@ -1,7 +1,7 @@
 [
   {
     "annotations": {
-      "__alertId__": "YCgu57m4z_017_Notification",
+      "__alertId__": "YCgu57m4z_017_High",
       "__dashboardUid__": "YCgu57m4z",
       "__panelId__": "17",
       "description": "\n\nWe ignore any previous one-off proposers to keep this metric at 0. Anything greater than 0 should be considered to be created by an attacker.",
@@ -37,7 +37,7 @@
             {
               "evaluator": {
                 "params": [
-                  null,
+                  1,
                   0
                 ],
                 "type": "gt"
@@ -77,16 +77,16 @@
       }
     ],
     "execErrState": "Alerting",
-    "folderUID": "ARwYXYzVk",
-    "for": "0",
-    "id": 220174,
+    "folderUID": "zpejR_iVz",
+    "for": "0s",
+    "id": 180171,
     "labels": {
-      "channel": "notification"
+      "channel": "high-alert"
     },
     "noDataState": "OK",
     "orgID": 1,
     "ruleGroup": "group",
-    "title": "Proposals by Unknown Proposers (Notification)",
-    "uid": "YCgu57m4z_017_Notification"
+    "title": "Proposals by Unknown Proposers (High)",
+    "uid": "YCgu57m4z_017_High"
   }
 ]
diff --git a/monitoring/grafana/alerts/audius-token_contract-balances-staking.json b/monitoring/grafana/alerts/audius-token_contract-balances-staking.json
@@ -1,7 +1,7 @@
 [
   {
     "annotations": {
-      "__alertId__": "jPaB_Yk4k_017_Notification",
+      "__alertId__": "jPaB_Yk4k_017_Low",
       "__dashboardUid__": "jPaB_Yk4k",
       "__panelId__": "17",
       "description": "\n\nIf this number dips, we may have lost a service provider.",
@@ -37,10 +37,10 @@
             {
               "evaluator": {
                 "params": [
-                  null,
+                  355000000,
                   0
                 ],
-                "type": "gt"
+                "type": "lt"
               },
               "operator": {
                 "type": "or"
@@ -77,16 +77,16 @@
       }
     ],
     "execErrState": "Alerting",
-    "folderUID": "ARwYXYzVk",
-    "for": "0",
-    "id": 200174,
+    "folderUID": "zpejR_iVz",
+    "for": "0s",
+    "id": 110173,
     "labels": {
-      "channel": "notification"
+      "channel": "low-alert"
     },
     "noDataState": "OK",
     "orgID": 1,
     "ruleGroup": "group",
-    "title": "Contract Balances (Staking) (Notification)",
-    "uid": "jPaB_Yk4k_017_Notification"
+    "title": "Contract Balances (Staking) (Low)",
+    "uid": "jPaB_Yk4k_017_Low"
   }
 ]