Add alertmanager support to backend target

CHANGELOG.md

@@ -10,7 +10,7 @@
 * [CHANGE] Ingester: removed deprecated `-blocks-storage.tsdb.isolation-enabled` option. TSDB-level isolation is now always disabled in Mimir. #2782
 * [CHANGE] Compactor: `-compactor.partial-block-deletion-delay` must either be set to 0 (to disable partial blocks deletion) or a value higher than `4h`. #2787
 * [FEATURE] Introduced an experimental anonymous usage statistics tracking (disabled by default), to help Mimir maintainers driving better decisions to support the opensource community. The tracking system anonymously collects non-sensitive and non-personal identifiable information about the running Mimir cluster, and is disabled by default. #2643 #2662 #2685 #2732 #2735
-* [FEATURE] Introduced an experimental deployment mode called read-write and running a fully featured Mimir cluster with three components: write, read and backend. The read-write deployment mode is a trade-off between the monolithic mode (only one component, no isolation) and the microservices mode (many components, high isolation). #2754
+* [FEATURE] Introduced an experimental deployment mode called read-write and running a fully featured Mimir cluster with three components: write, read and backend. The read-write deployment mode is a trade-off between the monolithic mode (only one component, no isolation) and the microservices mode (many components, high isolation). #2754 #2838
 * [FEATURE] Query-frontend: introduced experimental support to split instant queries by time. The instant query splitting can be enabled setting `-query-frontend.split-instant-queries-by-interval`. #2469 #2564 #2565 #2570 #2571 #2572 #2573 #2574 #2575 #2576 #2581 #2582 #2601 #2632 #2633 #2634 #2641 #2642 #2766
 * [ENHANCEMENT] Distributor: Add `cortex_distributor_query_ingester_chunks_deduped_total` and `cortex_distributor_query_ingester_chunks_total` metrics for determining how effective ingester chunk deduplication at query time is. #2713
 * [ENHANCEMENT] Upgrade Docker base images to `alpine:3.16.2`. #2729

development/mimir-read-write-mode/.data-minio/.gitignore

@@ -1,4 +1,5 @@
 *
 !mimir-ruler
 !mimir-blocks
+!mimir-alertmanager
 !.gitignore

development/mimir-read-write-mode/.data-minio/mimir-alertmanager/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

development/mimir-read-write-mode/config/alertmanager-fallback-config.yaml

@@ -0,0 +1,7 @@
+route:
+  group_wait: 0s
+  receiver: empty-receiver
+
+receivers:
+  # In this example we're not going to send any notification out of Alertmanager.
+  - name: 'empty-receiver'

development/mimir-read-write-mode/config/example-alertmanager.yaml

@@ -0,0 +1,16 @@
+# Example alertmanager config file to load to Mimir. It is used as the fallback configuration for the Alertmanager.
+global:
+  # The smarthost and SMTP sender used for mail notifications.
+  smtp_smarthost: 'localhost:25'
+  smtp_from: 'alertmanager@example.org'
+  smtp_auth_username: 'alertmanager'
+  smtp_auth_password: 'password'
+
+route:
+  # A default receiver.
+  receiver: send-email
+
+receivers:
+  - name: send-email
+    email_configs:
+      - to: 'someone@localhost'

development/mimir-read-write-mode/config/example-rules.yaml

@@ -0,0 +1,10 @@
+# Example rules file to load to Mimir via the ruler API.
+groups:
+  - name: alerts
+    rules:
+      - alert: AlwaysFiring
+        expr: count(up) >= 0
+        labels:
+          severity: page
+        annotations:
+          summary: This is an always-firing test alert.

development/mimir-read-write-mode/config/mimir.yaml

@@ -1,13 +1,17 @@
 multitenancy_enabled: false
 
+common:
+  storage:
+    backend: s3
+    s3:
+      endpoint:          minio:9000
+      access_key_id:     mimir
+      secret_access_key: supersecret
+      insecure:          true
+
 blocks_storage:
-  backend: s3
   s3:
-    endpoint:          minio:9000
     bucket_name:       mimir-blocks
-    access_key_id:     mimir
-    secret_access_key: supersecret
-    insecure:          true
   tsdb:
     dir: /data/ingester
 
@@ -34,15 +38,12 @@ memberlist:
 
 ruler:
   rule_path: /data/ruler
+  # Each ruler is configured to route alerts to the Alertmanager running within the same component.
+  alertmanager_url: http://mimir-backend-1:8006/alertmanager
 
 ruler_storage:
-  backend: s3
   s3:
     bucket_name:       mimir-ruler
-    endpoint:          minio:9000
-    access_key_id:     mimir
-    secret_access_key: supersecret
-    insecure:          true
 
 frontend:
   # Currently we can't specify multiple addresses, so we're just using a single replica for the query-scheduler.
@@ -54,5 +55,14 @@ frontend_worker:
   # See: https://github.com/grafana/mimir/issues/2012
   scheduler_address: "mimir-backend-1:9006"
 
+alertmanager:
+  data_dir: /data/alertmanager
+  fallback_config_file: ./config/alertmanager-fallback-config.yaml
+  external_url: http://localhost:8006/alertmanager
+
+alertmanager_storage:
+  s3:
+    bucket_name: mimir-alertmanager
+
 runtime_config:
   file: ./config/runtime.yaml

pkg/mimir/mimir.go

@@ -233,7 +233,7 @@ func (c *Config) Validate(log log.Logger) error {
 	if err := c.AlertmanagerStorage.Validate(); err != nil {
 		return errors.Wrap(err, "invalid alertmanager storage config")
 	}
-	if c.isModuleEnabled(AlertManager) {
+	if c.isAnyModuleEnabled(AlertManager, Backend) {
 		if err := c.Alertmanager.Validate(); err != nil {
 			return errors.Wrap(err, "invalid alertmanager config")
 		}
@@ -259,7 +259,7 @@ func (c *Config) validateBucketConfigs() error {
 	errs := multierror.New()
 
 	// Validate alertmanager bucket config.
-	if c.isAnyModuleEnabled(AlertManager) && c.AlertmanagerStorage.Backend != alertstorelocal.Name {
+	if c.isAnyModuleEnabled(AlertManager, Backend) && c.AlertmanagerStorage.Backend != alertstorelocal.Name {
 		errs.Add(errors.Wrap(validateBucketConfig(c.AlertmanagerStorage.Config, c.BlocksStorage.Bucket), "alertmanager storage"))
 	}
 

pkg/mimir/mimir_test.go

@@ -167,16 +167,16 @@ func TestMimir(t *testing.T) {
 		"-target=write": {
 			target:                  []string{Write},
 			expectedEnabledModules:  []string{DistributorService, IngesterService},
-			expectedDisabledModules: []string{Querier, Ruler, StoreGateway, Compactor},
+			expectedDisabledModules: []string{Querier, Ruler, StoreGateway, Compactor, AlertManager},
 		},
 		"-target=read": {
 			target:                  []string{Read},
 			expectedEnabledModules:  []string{QueryFrontend, Querier},
-			expectedDisabledModules: []string{IngesterService, Ruler, StoreGateway, Compactor},
+			expectedDisabledModules: []string{IngesterService, Ruler, StoreGateway, Compactor, AlertManager},
 		},
 		"-target=backend": {
 			target:                  []string{Backend},
-			expectedEnabledModules:  []string{QueryScheduler, Ruler, StoreGateway, Compactor},
+			expectedEnabledModules:  []string{QueryScheduler, Ruler, StoreGateway, Compactor, AlertManager},
 			expectedDisabledModules: []string{IngesterService, QueryFrontend, Querier},
 		},
 	}

pkg/mimir/modules.go

@@ -839,7 +839,7 @@ func (t *Mimir) setupModuleManager() error {
 		TenantFederation:         {Queryable},
 		Write:                    {Distributor, Ingester},
 		Read:                     {QueryFrontend, Querier},
-		Backend:                  {QueryScheduler, Ruler, StoreGateway, Compactor, OverridesExporter},
+		Backend:                  {QueryScheduler, Ruler, StoreGateway, Compactor, AlertManager, OverridesExporter},
 		All:                      {QueryFrontend, Querier, Ingester, Distributor, StoreGateway, Ruler, Compactor},
 	}
 	for mod, targets := range deps {

pkg/mimir/sanity_check.go

@@ -66,7 +66,7 @@ func checkDirectoriesReadWriteAccess(
 	if cfg.isAnyModuleEnabled(All, Ruler, Backend) {
 		errs.Add(errors.Wrap(checkDirReadWriteAccess(cfg.Ruler.RulePath, dirExistFn, isDirReadWritableFn), "ruler"))
 	}
-	if cfg.isAnyModuleEnabled(AlertManager) {
+	if cfg.isAnyModuleEnabled(AlertManager, Backend) {
 		errs.Add(errors.Wrap(checkDirReadWriteAccess(cfg.Alertmanager.DataDir, dirExistFn, isDirReadWritableFn), "alertmanager"))
 	}
 
@@ -133,7 +133,7 @@ func checkObjectStoresConfig(ctx context.Context, cfg Config, logger log.Logger)
 	}
 
 	// Check alertmanager storage config.
-	if cfg.isAnyModuleEnabled(AlertManager) && cfg.AlertmanagerStorage.Backend != alertstorelocal.Name {
+	if cfg.isAnyModuleEnabled(AlertManager, Backend) && cfg.AlertmanagerStorage.Backend != alertstorelocal.Name {
 		errs.Add(errors.Wrap(checkObjectStoreConfig(ctx, cfg.AlertmanagerStorage.Config, logger), "alertmanager storage"))
 	}