Changed -blocks-storage.tsdb.isolation-enabled default to false and deprecated config option

CHANGELOG.md

@@ -8,6 +8,7 @@
     - `-alertmanager.alertmanager-client.grpc-max-send-msg-size` now defaults to 100 MiB (previously was not configurable and set to 4 MiB)
     - `-alertmanager.max-recv-msg-size` now defaults to 100 MiB (previously was 16 MiB)
 * [CHANGE] Ingester: Add `user` label to metrics `cortex_ingester_ingested_samples_total` and `cortex_ingester_ingested_samples_failures_total`. #1533
+* [CHANGE] Ingester: Changed `-blocks-storage.tsdb.isolation-enabled` default from `true` to `false`. The config option has also been deprecated and will be removed in 2 minor version.
 * [CHANGE] Query-frontend: results cache keys are now versioned, this will cause cache to be re-filled when rolling out this version. #1631
 * [FEATURE] Ruler: Allow setting `evaluation_delay` for each rule group via rules group configuration file. #1474
 * [FEATURE] Ruler: Added support for expression remote evaluation. #1536

cmd/mimir/config-descriptor.json

@@ -5307,9 +5307,9 @@
               "kind": "field",
               "name": "isolation_enabled",
               "required": false,
-              "desc": "Enables TSDB isolation feature. Disabling may improve performance.",
+              "desc": "[Deprecated] Enables TSDB isolation feature. Disabling may improve performance.",
               "fieldValue": null,
-              "fieldDefaultValue": true,
+              "fieldDefaultValue": false,
               "fieldFlag": "blocks-storage.tsdb.isolation-enabled",
               "fieldType": "boolean",
               "fieldCategory": "advanced"

cmd/mimir/help-all.txt.tmpl

@@ -486,7 +486,7 @@ Usage of ./cmd/mimir/mimir:
   -blocks-storage.tsdb.head-compaction-interval duration
     	How frequently ingesters try to compact TSDB head. Block is only created if data covers smallest block range. Must be greater than 0 and max 5 minutes. (default 1m0s)
   -blocks-storage.tsdb.isolation-enabled
-    	Enables TSDB isolation feature. Disabling may improve performance. (default true)
+    	[Deprecated] Enables TSDB isolation feature. Disabling may improve performance.
   -blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup int
     	limit the number of concurrently opening TSDB's on startup (default 10)
   -blocks-storage.tsdb.memory-snapshot-on-shutdown

docs/sources/operators-guide/configuring/about-versioning.md

@@ -79,6 +79,9 @@ The following features are currently experimental:
 
 The following features are currently deprecated:
 
+- Ingester:
+  - `-blocks-storage.tsdb.isolation-enabled` CLI flag and `isolation_enabled` YAML config parameter. This will be removed in version 2.3.0.
+  - `active_series_custom_trackers` YAML config parameter in the ingester block. The configuration has been moved to limit config, the ingester config will be removed in version 2.3.0.
 - Ruler:
   - `/api/v1/rules/**` configuration endpoints. These will be removed in version 2.2.0. Use their `<prometheus-http-prefix>/config/v1/rules/**` equivalents instead.
   - `<prometheus-http-prefix>/rules/**` configuration endpoints. These will be removed in version 2.2.0. Use their `<prometheus-http-prefix>/config/v1/rules/**` equivalents instead.

docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md

@@ -3408,10 +3408,10 @@ tsdb:
   # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size
   [head_chunks_write_queue_size: <int> | default = 0]
 
-  # (advanced) Enables TSDB isolation feature. Disabling may improve
-  # performance.
+  # (advanced) [Deprecated] Enables TSDB isolation feature. Disabling may
+  # improve performance.
   # CLI flag: -blocks-storage.tsdb.isolation-enabled
-  [isolation_enabled: <boolean> | default = true]
+  [isolation_enabled: <boolean> | default = false]
 
   # (advanced) Max size - in bytes - of the in-memory series hash cache. The
   # cache is shared across all tenants and it's used only when query sharding is

operations/mimir-tests/test-defaults-generated.yaml

@@ -1004,7 +1004,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir-tests/test-disable-chunk-streaming-generated.yaml

@@ -1324,7 +1324,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir-tests/test-gossip-generated.yaml

@@ -1365,7 +1365,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir-tests/test-gossip-multi-zone-generated.yaml

@@ -1588,7 +1588,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000
@@ -1709,7 +1708,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000
@@ -1830,7 +1828,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir-tests/test-gossip-multikv-generated.yaml

@@ -1392,7 +1392,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml

@@ -1392,7 +1392,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml

@@ -1249,7 +1249,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir-tests/test-multi-zone-generated.yaml

@@ -1534,7 +1534,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000
@@ -1650,7 +1649,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000
@@ -1766,7 +1764,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml

@@ -1585,7 +1585,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000
@@ -1699,7 +1698,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000
@@ -1815,7 +1813,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000
@@ -1931,7 +1928,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir-tests/test-query-sharding-generated.yaml

@@ -1327,7 +1327,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir-tests/test-shuffle-sharding-generated.yaml

@@ -1333,7 +1333,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -distributor.ingestion-tenant-shard-size=3

operations/mimir-tests/test-storage-azure-generated.yaml

@@ -1335,7 +1335,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir-tests/test-storage-gcs-generated.yaml

@@ -1323,7 +1323,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir-tests/test-storage-s3-generated.yaml

@@ -1330,7 +1330,6 @@ spec:
         - -blocks-storage.tsdb.block-ranges-period=2h
         - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h
         - -blocks-storage.tsdb.dir=/data/tsdb
-        - -blocks-storage.tsdb.isolation-enabled=false
         - -blocks-storage.tsdb.ship-interval=1m
         - -distributor.health-check-ingesters=true
         - -ingester.max-global-series-per-metric=20000

operations/mimir/ingester.libsonnet

@@ -40,9 +40,6 @@
       // Close idle TSDBs.
       'blocks-storage.tsdb.close-idle-tsdb-timeout': $._config.queryConfig['querier.query-ingesters-within'],
 
-      // Disable TSDB isolation.
-      'blocks-storage.tsdb.isolation-enabled': 'false',
-
       // Persist ring tokens so that when the ingester will be restarted
       // it will pick the same tokens
       'ingester.ring.tokens-file-path': '/data/tokens',

pkg/storage/tsdb/config.go

@@ -157,7 +157,7 @@ type TSDBConfig struct {
 	CloseIdleTSDBTimeout      time.Duration `yaml:"close_idle_tsdb_timeout" category:"advanced"`
 	MemorySnapshotOnShutdown  bool          `yaml:"memory_snapshot_on_shutdown" category:"experimental"`
 	HeadChunksWriteQueueSize  int           `yaml:"head_chunks_write_queue_size" category:"experimental"`
-	IsolationEnabled          bool          `yaml:"isolation_enabled" category:"advanced"`
+	IsolationEnabled          bool          `yaml:"isolation_enabled" category:"advanced"` // TODO Remove in Mimir 2.3.0
 
 	// Series hash cache.
 	SeriesHashCacheMaxBytes uint64 `yaml:"series_hash_cache_max_size_bytes" category:"advanced"`
@@ -198,7 +198,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) {
 	f.DurationVar(&cfg.CloseIdleTSDBTimeout, "blocks-storage.tsdb.close-idle-tsdb-timeout", 13*time.Hour, "If TSDB has not received any data for this duration, and all blocks from TSDB have been shipped, TSDB is closed and deleted from local disk. If set to positive value, this value should be equal or higher than -querier.query-ingesters-within flag to make sure that TSDB is not closed prematurely, which could cause partial query results. 0 or negative value disables closing of idle TSDB.")
 	f.BoolVar(&cfg.MemorySnapshotOnShutdown, "blocks-storage.tsdb.memory-snapshot-on-shutdown", false, "True to enable snapshotting of in-memory TSDB data on disk when shutting down.")
 	f.IntVar(&cfg.HeadChunksWriteQueueSize, "blocks-storage.tsdb.head-chunks-write-queue-size", 0, "The size of the write queue used by the head chunks mapper. Lower values reduce memory utilisation at the cost of potentially higher ingest latency. Value of 0 switches chunks mapper to implementation without a queue.")
-	f.BoolVar(&cfg.IsolationEnabled, "blocks-storage.tsdb.isolation-enabled", true, "Enables TSDB isolation feature. Disabling may improve performance.")
+	f.BoolVar(&cfg.IsolationEnabled, "blocks-storage.tsdb.isolation-enabled", false, "[Deprecated] Enables TSDB isolation feature. Disabling may improve performance.")
 }
 
 // Validate the config.