forked from etcd-io/website
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request etcd-io#600 from spzala/updatealertrules
Add alert rules
- Loading branch information
Showing
1 changed file
with
148 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,148 @@ | ||
| # these rules synced manually from https://github.com/etcd-io/etcd/blob/release-3.5/contrib/mixin/mixin.libsonnet | ||
| groups: | ||
| - name: etcd | ||
| rules: | ||
| - alert: etcdMembersDown | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": members are down ({{ $value }}).' | ||
| summary: etcd cluster members are down. | ||
| expr: | | ||
| max without (endpoint) ( | ||
| sum without (instance) (up{job=~".*etcd.*"} == bool 0) | ||
| or | ||
| count without (To) ( | ||
| sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 | ||
| ) | ||
| ) | ||
| > 0 | ||
| for: 10m | ||
| labels: | ||
| severity: critical | ||
| - alert: etcdInsufficientMembers | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": insufficient members ({{ $value }}).' | ||
| summary: etcd cluster has insufficient number of members. | ||
| expr: | | ||
| sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2) | ||
| for: 3m | ||
| labels: | ||
| severity: critical | ||
| - alert: etcdNoLeader | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": member {{ $labels.instance }} has no leader.' | ||
| summary: etcd cluster has no leader. | ||
| expr: | | ||
| etcd_server_has_leader{job=~".*etcd.*"} == 0 | ||
| for: 1m | ||
| labels: | ||
| severity: critical | ||
| - alert: etcdHighNumberOfLeaderChanges | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": {{ $value }} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' | ||
| summary: etcd cluster has high number of leader changes. | ||
| expr: | | ||
| increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4 | ||
| for: 5m | ||
| labels: | ||
| severity: warning | ||
| - alert: etcdHighNumberOfFailedGRPCRequests | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.' | ||
| summary: etcd cluster has high number of failed grpc requests. | ||
| expr: | | ||
| 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) | ||
| / | ||
| sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) | ||
| > 1 | ||
| for: 10m | ||
| labels: | ||
| severity: warning | ||
| - alert: etcdHighNumberOfFailedGRPCRequests | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.' | ||
| summary: etcd cluster has high number of failed grpc requests. | ||
| expr: | | ||
| 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) | ||
| / | ||
| sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) | ||
| > 5 | ||
| for: 5m | ||
| labels: | ||
| severity: critical | ||
| - alert: etcdGRPCRequestsSlow | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method.' | ||
| summary: etcd grpc requests are slow | ||
| expr: | | ||
| histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) | ||
| > 0.15 | ||
| for: 10m | ||
| labels: | ||
| severity: critical | ||
| - alert: etcdMemberCommunicationSlow | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": member communication with {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.' | ||
| summary: etcd cluster member communication is slow. | ||
| expr: | | ||
| histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) | ||
| > 0.15 | ||
| for: 10m | ||
| labels: | ||
| severity: warning | ||
| - alert: etcdHighNumberOfFailedProposals | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures within the last 30 minutes on etcd instance {{ $labels.instance }}.' | ||
| summary: etcd cluster has high number of proposal failures. | ||
| expr: | | ||
| rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 | ||
| for: 15m | ||
| labels: | ||
| severity: warning | ||
| - alert: etcdHighFsyncDurations | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.' | ||
| summary: etcd cluster 99th percentile fsync durations are too high. | ||
| expr: | | ||
| histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) | ||
| > 0.5 | ||
| for: 10m | ||
| labels: | ||
| severity: warning | ||
| - alert: etcdHighFsyncDurations | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.' | ||
| summary: etcd cluster 99th percentile fsync durations are too high. | ||
| expr: | | ||
| histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) | ||
| > 1 | ||
| for: 10m | ||
| labels: | ||
| severity: critical | ||
| - alert: etcdHighCommitDurations | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.' | ||
| summary: etcd cluster 99th percentile commit durations are too high. | ||
| expr: | | ||
| histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) | ||
| > 0.25 | ||
| for: 10m | ||
| labels: | ||
| severity: warning | ||
| - alert: etcdBackendQuotaLowSpace | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": database size exceeds the defined quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' | ||
| summary: etcd cluster database is running full. | ||
| expr: | | ||
| (etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100 > 95 | ||
| for: 10m | ||
| labels: | ||
| severity: critical | ||
| - alert: etcdExcessiveDatabaseGrowth | ||
| annotations: | ||
| description: 'etcd cluster "{{ $labels.job }}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive.' | ||
| summary: etcd cluster database growing very fast. | ||
| expr: | | ||
| predict_linear(etcd_mvcc_db_total_size_in_bytes[4h], 4*60*60) > etcd_server_quota_backend_bytes | ||
| for: 10m | ||
| labels: | ||
| severity: warning |