Merge pull request #3353 from alphagov/ris-shield-cloudwatch

AWS Shield CloudWatch metrics and alertmanager alerts

config/cloudwatch-exporter/config.yml

@@ -64,3 +64,33 @@ discovery:
       - Rule
       - Region
       addCloudwatchTimestamp: true
+    - type: "AWS/DDoSProtection"
+      regions:
+        - (( grab $AWS_REGION ))
+      # can't currently filter DDoSProtection by tag, region
+      # will have to do
+      metrics:
+        - name: DDoSDetected
+          statistics:
+            - Maximum
+          period: 60
+          length: 600
+      dimensionNameRequirements:
+      - ResourceArn
+      addCloudwatchTimestamp: true
+    - type: "AWS/DDoSProtection"
+      regions:
+        - (( grab $AWS_REGION ))
+      # can't currently filter DDoSProtection by tag, region
+      # will have to do
+      metrics:
+        - name: VolumePacketsPerSecond
+          statistics:
+            - Average
+            - Maximum
+          period: 60
+          length: 600
+      dimensionNameRequirements:
+      - ResourceArn
+      - MitigationAction
+      addCloudwatchTimestamp: true

manifests/prometheus/alerts.d/shield-attacks.yml

@@ -1,15 +1,37 @@
 # Source: paas-metrics
+# Source: yet-another-cloudwatch-exporter
 ---
 
 - type: replace
   path: /instance_groups/name=prometheus2/jobs/name=prometheus2/properties/prometheus/custom_rules?/-
   value:
-    name: ShieldOngoingAttacks
+    name: ShieldAttacks
     rules:
       - alert: ShieldAnyOngoingAttacks
         expr: paas_aws_shield_ongoing_attacks > 0
         labels:
           severity: warning
+          service: elb
         annotations:
           summary: "AWS Shield reporting an ongoing attack"
           description: "An ongoing attack being reported by AWS Shield could be indicative of a large traffic spike. Is a tenant load testing?"
+
+      - alert: DDoSDetected
+        # weird metric name comes from yet-another-cloudwatch-exporter's
+        # capitalization-based auto-snake-case-conversion
+        expr: aws_ddosprotection_ddo_sdetected_maximum{dimension_ResourceArn=~".*/((metrics_environment))-cf-rtr-.*"} > 0
+        labels:
+          severity: warning
+          service: elb
+        annotations:
+          summary: "AWS has detected a DDoS attack"
+          description: "AWS has detected a DDoS attack affecting {{$labels.dimension_ResourceArn}}"
+
+      - alert: DDoSBeingMitigated
+        expr: aws_ddosprotection_volume_packets_per_second_average{dimension_ResourceArn=~".*/((metrics_environment))-cf-rtr-.*", dimension_MitigationAction="Drop"} > 0
+        labels:
+          severity: critical
+          service: elb
+        annotations:
+          summary: "AWS is mitigating a DDoS attack"
+          description: "AWS is mitigating a DDoS attack affecting {{$labels.dimension_ResourceArn}} - you should probably check it isn't preventing genuine traffic getting through."

manifests/prometheus/alerts.d/wafv2.yml

@@ -7,9 +7,7 @@
     name: WafThrottlingIPsMaxRate
     rules:
       - alert: WafThrottlingIPsMaxRate
-        # the cloudwatch exporter should already restrict the metrics available
-        # to those of our deploy_env
-        expr: (aws_wafv2_blocked_requests_maximum{dimension_WebACL=~".+-rtr-lbs-web-acl$", dimension_Rule=~".+-rtr-lbs-max-request-rate-xff-blocked$"} > 0) or (aws_wafv2_blocked_requests_maximum{dimension_WebACL=~".+-rtr-lbs-web-acl$", dimension_Rule=~".+-rtr-lbs-max-request-rate-direct-blocked$"} > 0)
+        expr: (aws_wafv2_blocked_requests_maximum{dimension_WebACL="((metrics_environment))-rtr-lbs-web-acl", dimension_Rule="((metrics_environment))-rtr-lbs-max-request-rate-xff-blocked"} > 0) or (aws_wafv2_blocked_requests_maximum{dimension_WebACL="((metrics_environment))-rtr-lbs-web-acl", dimension_Rule="((metrics_environment))-rtr-lbs-max-request-rate-direct-blocked"} > 0)
         labels:
           severity: warning
           service: elb

manifests/prometheus/spec/alerts/shield-attacks.test.yml

@@ -0,0 +1,58 @@
+---
+rule_files:
+  # See alerts_validation_spec.rb for details of how stdin gets set:
+  - fixtures/rules.yml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 5m
+    input_series:
+      - series: 'paas_aws_shield_ongoing_attacks'
+        values: 1
+
+    alert_rule_test:
+      - alertname: ShieldAnyOngoingAttacks
+        eval_time: 5m
+        exp_alerts:
+          - exp_annotations:
+              summary: "AWS Shield reporting an ongoing attack"
+              description: "An ongoing attack being reported by AWS Shield could be indicative of a large traffic spike. Is a tenant load testing?"
+            exp_labels:
+              severity: warning
+              service: elb
+
+  - interval: 5m
+    input_series:
+      - series: 'aws_ddosprotection_ddo_sdetected_maximum{dimension_ResourceArn="arn:aws:elasticloadbalancing:eu-west-1:999999999999:loadbalancer/app/test-cf-rtr-sys/fefefefefefefefe"}'
+        values: 123
+
+    alert_rule_test:
+      - alertname: DDoSDetected
+        eval_time: 5m
+        exp_alerts:
+          - exp_annotations:
+              summary: "AWS has detected a DDoS attack"
+              description: "AWS has detected a DDoS attack affecting arn:aws:elasticloadbalancing:eu-west-1:999999999999:loadbalancer/app/test-cf-rtr-sys/fefefefefefefefe"
+            exp_labels:
+              severity: warning
+              service: elb
+              dimension_ResourceArn: "arn:aws:elasticloadbalancing:eu-west-1:999999999999:loadbalancer/app/test-cf-rtr-sys/fefefefefefefefe"
+
+  - interval: 5m
+    input_series:
+      - series: 'aws_ddosprotection_volume_packets_per_second_average{dimension_ResourceArn="arn:aws:elasticloadbalancing:eu-west-1:999999999999:loadbalancer/app/test-cf-rtr-sys/fefefefefefefefe", dimension_MitigationAction="Drop"}'
+        values: 123
+
+    alert_rule_test:
+      - alertname: DDoSBeingMitigated
+        eval_time: 5m
+        exp_alerts:
+          - exp_annotations:
+              summary: "AWS is mitigating a DDoS attack"
+              description: "AWS is mitigating a DDoS attack affecting arn:aws:elasticloadbalancing:eu-west-1:999999999999:loadbalancer/app/test-cf-rtr-sys/fefefefefefefefe - you should probably check it isn't preventing genuine traffic getting through."
+            exp_labels:
+              severity: critical
+              service: elb
+              dimension_ResourceArn: "arn:aws:elasticloadbalancing:eu-west-1:999999999999:loadbalancer/app/test-cf-rtr-sys/fefefefefefefefe"
+              dimension_MitigationAction: "Drop"

manifests/prometheus/spec/alerts/waf-throttling-ips-max-rate.test.yml

@@ -8,9 +8,9 @@ evaluation_interval: 1m
 tests:
   - interval: 5m
     input_series:
-      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="foo-rtr-lbs-web-acl",dimension_Rule="foo-rtr-lbs-max-request-rate-xff-blocked"}'
+      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="test-rtr-lbs-web-acl",dimension_Rule="test-rtr-lbs-max-request-rate-xff-blocked"}'
         values: 123
-      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="foo-rtr-lbs-web-acl",dimension_Rule="foo-rtr-lbs-max-request-rate-direct-blocked"}'
+      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="test-rtr-lbs-web-acl",dimension_Rule="test-rtr-lbs-max-request-rate-direct-blocked"}'
         values: _
 
     alert_rule_test:
@@ -23,14 +23,14 @@ tests:
             exp_labels:
               severity: warning
               service: elb
-              dimension_WebACL: "foo-rtr-lbs-web-acl"
-              dimension_Rule: "foo-rtr-lbs-max-request-rate-xff-blocked"
+              dimension_WebACL: "test-rtr-lbs-web-acl"
+              dimension_Rule: "test-rtr-lbs-max-request-rate-xff-blocked"
 
   - interval: 5m
     input_series:
-      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="foo-rtr-lbs-web-acl",dimension_Rule="foo-rtr-lbs-max-request-rate-xff-blocked"}'
+      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="test-rtr-lbs-web-acl",dimension_Rule="test-rtr-lbs-max-request-rate-xff-blocked"}'
         values: _
-      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="foo-rtr-lbs-web-acl",dimension_Rule="foo-rtr-lbs-max-request-rate-direct-blocked"}'
+      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="test-rtr-lbs-web-acl",dimension_Rule="test-rtr-lbs-max-request-rate-direct-blocked"}'
         values: 123
 
     alert_rule_test:
@@ -43,16 +43,16 @@ tests:
             exp_labels:
               severity: warning
               service: elb
-              dimension_WebACL: "foo-rtr-lbs-web-acl"
-              dimension_Rule: "foo-rtr-lbs-max-request-rate-direct-blocked"
+              dimension_WebACL: "test-rtr-lbs-web-acl"
+              dimension_Rule: "test-rtr-lbs-max-request-rate-direct-blocked"
 
   - interval: 5m
     input_series:
-      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="foo-rtr-lbs-web-acl",dimension_Rule="foo-rtr-lbs-max-request-rate-xff-blocked"}'
+      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="test-rtr-lbs-web-acl",dimension_Rule="test-rtr-lbs-max-request-rate-xff-blocked"}'
         values: 0
-      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="foo-rtr-lbs-web-acl",dimension_Rule="foo-rtr-lbs-max-request-rate-direct-blocked"}'
+      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="test-rtr-lbs-web-acl",dimension_Rule="test-rtr-lbs-max-request-rate-direct-blocked"}'
         values: _
-      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="foo-rtr-lbs-web-acl-bar",dimension_Rule="foo-rtr-lbs-max-request-rate-direct-blocked"}'
+      - series: 'aws_wafv2_blocked_requests_maximum{dimension_WebACL="test-rtr-lbs-web-acl-bar",dimension_Rule="test-rtr-lbs-max-request-rate-direct-blocked"}'
         values: 111
 
     alert_rule_test: []

terraform/cloudfoundry/lbs.tf

@@ -10,6 +10,10 @@ resource "aws_lb" "cf_loggregator" {
     prefix  = "cf-loggregator"
     enabled = true
   }
+
+  tags = {
+    deploy_env = var.env
+  }
 }
 
 resource "aws_lb_target_group" "cf_loggregator_rlp" {
@@ -48,6 +52,10 @@ resource "aws_lb_listener" "cf_loggregator" {
       status_code  = "404"
     }
   }
+
+  tags = {
+    deploy_env = var.env
+  }
 }
 
 resource "aws_lb_listener_rule" "cf_loggregator_rlp_log_api" {
@@ -64,6 +72,10 @@ resource "aws_lb_listener_rule" "cf_loggregator_rlp_log_api" {
       values = ["log-api.*"]
     }
   }
+
+  tags = {
+    deploy_env = var.env
+  }
 }
 
 resource "aws_lb_listener_rule" "cf_loggregator_rlp_log_stream" {
@@ -80,6 +92,10 @@ resource "aws_lb_listener_rule" "cf_loggregator_rlp_log_stream" {
       values = ["log-stream.*"]
     }
   }
+
+  tags = {
+    deploy_env = var.env
+  }
 }
 
 resource "aws_lb_listener_rule" "cf_doppler" {
@@ -96,6 +112,10 @@ resource "aws_lb_listener_rule" "cf_doppler" {
       values = ["doppler.*"]
     }
   }
+
+  tags = {
+    deploy_env = var.env
+  }
 }
 
 resource "aws_lb_target_group" "cf_doppler" {
@@ -131,6 +151,10 @@ resource "aws_lb" "cf_router_app_domain" {
     prefix  = "cf-rtr-apps"
     enabled = true
   }
+
+  tags = {
+    deploy_env = var.env
+  }
 }
 
 resource "aws_lb_listener" "cf_router_app_domain_http" {
@@ -149,6 +173,10 @@ resource "aws_lb_listener" "cf_router_app_domain_http" {
       query       = ""
     }
   }
+
+  tags = {
+    deploy_env = var.env
+  }
 }
 
 resource "aws_lb_listener" "cf_router_app_domain_https" {
@@ -162,6 +190,10 @@ resource "aws_lb_listener" "cf_router_app_domain_https" {
     type             = "forward"
     target_group_arn = aws_lb_target_group.cf_router_app_domain_https.arn
   }
+
+  tags = {
+    deploy_env = var.env
+  }
 }
 
 resource "aws_lb_target_group" "cf_router_app_domain_https" {
@@ -224,6 +256,10 @@ resource "aws_lb" "cf_router_system_domain" {
     prefix  = "cf-rtr-sys"
     enabled = true
   }
+
+  tags = {
+    deploy_env = var.env
+  }
 }
 
 resource "aws_lb_listener" "cf_router_system_domain_https" {
@@ -237,6 +273,10 @@ resource "aws_lb_listener" "cf_router_system_domain_https" {
     type             = "forward"
     target_group_arn = aws_lb_target_group.cf_router_system_domain_https.arn
   }
+
+  tags = {
+    deploy_env = var.env
+  }
 }
 
 resource "aws_lb_listener_certificate" "cf_router_metrics_domain_https" {