From 535aea521bf1b1cef2d0610fc03415f11e109f50 Mon Sep 17 00:00:00 2001
From: Robert Gildein <gildeinrobert@gmail.com>
Date: Tue, 20 Aug 2024 15:28:24 +0200
Subject: [PATCH] Add alert rules to katib-controller based on the KF093 spec
 (#231)

* Add alert rules to katib-controller based on the KF093 spec

* Delete charms/katib-controller/src/prometheus_alert_rules/unit_unavailable.rule
---
 .../KubeflowKatibControllerServices.rules     | 24 +++++++++++++++++++
 .../unit_unavailable.rule                     | 10 --------
 2 files changed, 24 insertions(+), 10 deletions(-)
 create mode 100644 charms/katib-controller/src/prometheus_alert_rules/KubeflowKatibControllerServices.rules
 delete mode 100644 charms/katib-controller/src/prometheus_alert_rules/unit_unavailable.rule

diff --git a/charms/katib-controller/src/prometheus_alert_rules/KubeflowKatibControllerServices.rules b/charms/katib-controller/src/prometheus_alert_rules/KubeflowKatibControllerServices.rules
new file mode 100644
index 00000000..211149f6
--- /dev/null
+++ b/charms/katib-controller/src/prometheus_alert_rules/KubeflowKatibControllerServices.rules
@@ -0,0 +1,24 @@
+groups:
+- name: KubeflowKatibControllerServices
+  rules:
+  - alert: KubeflowServiceDown
+    expr: up{} < 1
+    for: 5m
+    labels:
+      severity: critical
+    annotations:
+      summary: "{{ $labels.juju_charm }} service is Down ({{ $labels.juju_model }}/{{ $labels.juju_unit }})"
+      description: |
+       One or more targets of {{ $labels.juju_charm }} charm are down on unit {{ $labels.juju_model }}/{{ $labels.juju_unit }}.
+       LABELS = {{ $labels }}
+
+  - alert: KubeflowServiceIsNotStable
+    expr: avg_over_time(up{}[10m]) < 0.5
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ $labels.juju_charm }} service is not stable ({{ $labels.juju_model }}/{{ $labels.juju_unit }})"
+      description: |
+        {{ $labels.juju_charm }} unit {{ $labels.juju_model }}/{{ $labels.juju_unit }} has been unreachable at least 50% of the time over the last 10 minutes.
+        LABELS = {{ $labels }}
diff --git a/charms/katib-controller/src/prometheus_alert_rules/unit_unavailable.rule b/charms/katib-controller/src/prometheus_alert_rules/unit_unavailable.rule
deleted file mode 100644
index 337986b8..00000000
--- a/charms/katib-controller/src/prometheus_alert_rules/unit_unavailable.rule
+++ /dev/null
@@ -1,10 +0,0 @@
-alert: KatibControllerUnitIsUnavailable
-expr: up < 1
-for: 0m
-labels:
-  severity: critical
-annotations:
-  summary: Katib-controller unit {{ $labels.juju_model }}/{{ $labels.juju_unit }} unavailable
-  description: >
-    The Katib-controller unit {{ $labels.juju_model }} {{ $labels.juju_unit }} is unavailable
-    LABELS = {{ $labels }}