From f40fff7d57dea2a45ca2c67c1ca98ddb006119f6 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Thu, 7 Apr 2022 16:49:59 +0200 Subject: [PATCH] Changed -blocks-storage.tsdb.isolation-enabled default to false and deprecated config option Signed-off-by: Marco Pracucci --- CHANGELOG.md | 1 + cmd/mimir/config-descriptor.json | 4 ++-- cmd/mimir/help-all.txt.tmpl | 2 +- .../sources/operators-guide/configuring/about-versioning.md | 3 +++ .../configuring/reference-configuration-parameters/index.md | 6 +++--- operations/mimir-tests/test-defaults-generated.yaml | 1 - .../mimir-tests/test-disable-chunk-streaming-generated.yaml | 1 - operations/mimir-tests/test-gossip-generated.yaml | 1 - .../mimir-tests/test-gossip-multi-zone-generated.yaml | 3 --- operations/mimir-tests/test-gossip-multikv-generated.yaml | 1 - ...t-gossip-multikv-switch-primary-secondary-generated.yaml | 1 - .../mimir-tests/test-gossip-ruler-disabled-generated.yaml | 1 - operations/mimir-tests/test-multi-zone-generated.yaml | 3 --- .../test-multi-zone-with-ongoing-migration-generated.yaml | 4 ---- operations/mimir-tests/test-query-sharding-generated.yaml | 1 - operations/mimir-tests/test-shuffle-sharding-generated.yaml | 1 - operations/mimir-tests/test-storage-azure-generated.yaml | 1 - operations/mimir-tests/test-storage-gcs-generated.yaml | 1 - operations/mimir-tests/test-storage-s3-generated.yaml | 1 - operations/mimir/ingester.libsonnet | 3 --- pkg/storage/tsdb/config.go | 4 ++-- 21 files changed, 12 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29459b231cc..8efb33cf018 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - `-alertmanager.alertmanager-client.grpc-max-send-msg-size` now defaults to 100 MiB (previously was not configurable and set to 4 MiB) - `-alertmanager.max-recv-msg-size` now defaults to 100 MiB (previously was 16 MiB) * [CHANGE] Ingester: Add `user` label to metrics `cortex_ingester_ingested_samples_total` and `cortex_ingester_ingested_samples_failures_total`. #1533 +* [CHANGE] Ingester: Changed `-blocks-storage.tsdb.isolation-enabled` default from `true` to `false`. The config option has also been deprecated and will be removed in 2 minor version. * [CHANGE] Query-frontend: results cache keys are now versioned, this will cause cache to be re-filled when rolling out this version. #1631 * [FEATURE] Ruler: Allow setting `evaluation_delay` for each rule group via rules group configuration file. #1474 * [FEATURE] Ruler: Added support for expression remote evaluation. #1536 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index be82653236d..48b24e6547e 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -5307,9 +5307,9 @@ "kind": "field", "name": "isolation_enabled", "required": false, - "desc": "Enables TSDB isolation feature. Disabling may improve performance.", + "desc": "[Deprecated] Enables TSDB isolation feature. Disabling may improve performance.", "fieldValue": null, - "fieldDefaultValue": true, + "fieldDefaultValue": false, "fieldFlag": "blocks-storage.tsdb.isolation-enabled", "fieldType": "boolean", "fieldCategory": "advanced" diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 9ef69e6f961..3fc0b863503 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -486,7 +486,7 @@ Usage of ./cmd/mimir/mimir: -blocks-storage.tsdb.head-compaction-interval duration How frequently ingesters try to compact TSDB head. Block is only created if data covers smallest block range. Must be greater than 0 and max 5 minutes. (default 1m0s) -blocks-storage.tsdb.isolation-enabled - Enables TSDB isolation feature. Disabling may improve performance. (default true) + [Deprecated] Enables TSDB isolation feature. Disabling may improve performance. -blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup int limit the number of concurrently opening TSDB's on startup (default 10) -blocks-storage.tsdb.memory-snapshot-on-shutdown diff --git a/docs/sources/operators-guide/configuring/about-versioning.md b/docs/sources/operators-guide/configuring/about-versioning.md index 712c870b130..8daf1384c63 100644 --- a/docs/sources/operators-guide/configuring/about-versioning.md +++ b/docs/sources/operators-guide/configuring/about-versioning.md @@ -79,6 +79,9 @@ The following features are currently experimental: The following features are currently deprecated: +- Ingester: + - `-blocks-storage.tsdb.isolation-enabled` CLI flag and `isolation_enabled` YAML config parameter. This will be removed in version 2.3.0. + - `active_series_custom_trackers` YAML config parameter in the ingester block. The configuration has been moved to limit config, the ingester config will be removed in version 2.3.0. - Ruler: - `/api/v1/rules/**` configuration endpoints. These will be removed in version 2.2.0. Use their `/config/v1/rules/**` equivalents instead. - `/rules/**` configuration endpoints. These will be removed in version 2.2.0. Use their `/config/v1/rules/**` equivalents instead. diff --git a/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md b/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md index 2c13a83b34e..4d90447d9ae 100644 --- a/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md +++ b/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md @@ -3408,10 +3408,10 @@ tsdb: # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: | default = 0] - # (advanced) Enables TSDB isolation feature. Disabling may improve - # performance. + # (advanced) [Deprecated] Enables TSDB isolation feature. Disabling may + # improve performance. # CLI flag: -blocks-storage.tsdb.isolation-enabled - [isolation_enabled: | default = true] + [isolation_enabled: | default = false] # (advanced) Max size - in bytes - of the in-memory series hash cache. The # cache is shared across all tenants and it's used only when query sharding is diff --git a/operations/mimir-tests/test-defaults-generated.yaml b/operations/mimir-tests/test-defaults-generated.yaml index 63a92bfae5e..01fb0dc5ccd 100644 --- a/operations/mimir-tests/test-defaults-generated.yaml +++ b/operations/mimir-tests/test-defaults-generated.yaml @@ -1004,7 +1004,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml b/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml index f7ae2a0b392..b212a25a7e1 100644 --- a/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml +++ b/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml @@ -1324,7 +1324,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir-tests/test-gossip-generated.yaml b/operations/mimir-tests/test-gossip-generated.yaml index 7f801987275..43db92f4fd3 100644 --- a/operations/mimir-tests/test-gossip-generated.yaml +++ b/operations/mimir-tests/test-gossip-generated.yaml @@ -1365,7 +1365,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir-tests/test-gossip-multi-zone-generated.yaml b/operations/mimir-tests/test-gossip-multi-zone-generated.yaml index 75658bf1ed9..c6d492d49cf 100644 --- a/operations/mimir-tests/test-gossip-multi-zone-generated.yaml +++ b/operations/mimir-tests/test-gossip-multi-zone-generated.yaml @@ -1588,7 +1588,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 @@ -1709,7 +1708,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 @@ -1830,7 +1828,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir-tests/test-gossip-multikv-generated.yaml b/operations/mimir-tests/test-gossip-multikv-generated.yaml index 760f4dba72d..ab718878b42 100644 --- a/operations/mimir-tests/test-gossip-multikv-generated.yaml +++ b/operations/mimir-tests/test-gossip-multikv-generated.yaml @@ -1392,7 +1392,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml b/operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml index c63e61d2dfd..62e12127615 100644 --- a/operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml +++ b/operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml @@ -1392,7 +1392,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml b/operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml index d7c6c8a2a4f..902a4cbf33d 100644 --- a/operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml +++ b/operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml @@ -1249,7 +1249,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir-tests/test-multi-zone-generated.yaml b/operations/mimir-tests/test-multi-zone-generated.yaml index cdaedb1b614..06a60c2b328 100644 --- a/operations/mimir-tests/test-multi-zone-generated.yaml +++ b/operations/mimir-tests/test-multi-zone-generated.yaml @@ -1534,7 +1534,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 @@ -1650,7 +1649,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 @@ -1766,7 +1764,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml b/operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml index 387c4fdd1a0..00b7931c9b4 100644 --- a/operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml +++ b/operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml @@ -1585,7 +1585,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 @@ -1699,7 +1698,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 @@ -1815,7 +1813,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 @@ -1931,7 +1928,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir-tests/test-query-sharding-generated.yaml b/operations/mimir-tests/test-query-sharding-generated.yaml index 386ed218290..44814fc3234 100644 --- a/operations/mimir-tests/test-query-sharding-generated.yaml +++ b/operations/mimir-tests/test-query-sharding-generated.yaml @@ -1327,7 +1327,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir-tests/test-shuffle-sharding-generated.yaml b/operations/mimir-tests/test-shuffle-sharding-generated.yaml index 9c24cd61d71..c192d8895ef 100644 --- a/operations/mimir-tests/test-shuffle-sharding-generated.yaml +++ b/operations/mimir-tests/test-shuffle-sharding-generated.yaml @@ -1333,7 +1333,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -distributor.ingestion-tenant-shard-size=3 diff --git a/operations/mimir-tests/test-storage-azure-generated.yaml b/operations/mimir-tests/test-storage-azure-generated.yaml index 169fa432d88..d7653a063a5 100644 --- a/operations/mimir-tests/test-storage-azure-generated.yaml +++ b/operations/mimir-tests/test-storage-azure-generated.yaml @@ -1335,7 +1335,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir-tests/test-storage-gcs-generated.yaml b/operations/mimir-tests/test-storage-gcs-generated.yaml index 31bad9359a4..962a3fec0f6 100644 --- a/operations/mimir-tests/test-storage-gcs-generated.yaml +++ b/operations/mimir-tests/test-storage-gcs-generated.yaml @@ -1323,7 +1323,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir-tests/test-storage-s3-generated.yaml b/operations/mimir-tests/test-storage-s3-generated.yaml index 6fc0de89b21..397725ebdfb 100644 --- a/operations/mimir-tests/test-storage-s3-generated.yaml +++ b/operations/mimir-tests/test-storage-s3-generated.yaml @@ -1330,7 +1330,6 @@ spec: - -blocks-storage.tsdb.block-ranges-period=2h - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - - -blocks-storage.tsdb.isolation-enabled=false - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true - -ingester.max-global-series-per-metric=20000 diff --git a/operations/mimir/ingester.libsonnet b/operations/mimir/ingester.libsonnet index b9a8014bebb..50b19cfaa61 100644 --- a/operations/mimir/ingester.libsonnet +++ b/operations/mimir/ingester.libsonnet @@ -40,9 +40,6 @@ // Close idle TSDBs. 'blocks-storage.tsdb.close-idle-tsdb-timeout': $._config.queryConfig['querier.query-ingesters-within'], - // Disable TSDB isolation. - 'blocks-storage.tsdb.isolation-enabled': 'false', - // Persist ring tokens so that when the ingester will be restarted // it will pick the same tokens 'ingester.ring.tokens-file-path': '/data/tokens', diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index caa7491afc7..e769ca12fa4 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -157,7 +157,7 @@ type TSDBConfig struct { CloseIdleTSDBTimeout time.Duration `yaml:"close_idle_tsdb_timeout" category:"advanced"` MemorySnapshotOnShutdown bool `yaml:"memory_snapshot_on_shutdown" category:"experimental"` HeadChunksWriteQueueSize int `yaml:"head_chunks_write_queue_size" category:"experimental"` - IsolationEnabled bool `yaml:"isolation_enabled" category:"advanced"` + IsolationEnabled bool `yaml:"isolation_enabled" category:"advanced"` // TODO Remove in Mimir 2.3.0 // Series hash cache. SeriesHashCacheMaxBytes uint64 `yaml:"series_hash_cache_max_size_bytes" category:"advanced"` @@ -198,7 +198,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.CloseIdleTSDBTimeout, "blocks-storage.tsdb.close-idle-tsdb-timeout", 13*time.Hour, "If TSDB has not received any data for this duration, and all blocks from TSDB have been shipped, TSDB is closed and deleted from local disk. If set to positive value, this value should be equal or higher than -querier.query-ingesters-within flag to make sure that TSDB is not closed prematurely, which could cause partial query results. 0 or negative value disables closing of idle TSDB.") f.BoolVar(&cfg.MemorySnapshotOnShutdown, "blocks-storage.tsdb.memory-snapshot-on-shutdown", false, "True to enable snapshotting of in-memory TSDB data on disk when shutting down.") f.IntVar(&cfg.HeadChunksWriteQueueSize, "blocks-storage.tsdb.head-chunks-write-queue-size", 0, "The size of the write queue used by the head chunks mapper. Lower values reduce memory utilisation at the cost of potentially higher ingest latency. Value of 0 switches chunks mapper to implementation without a queue.") - f.BoolVar(&cfg.IsolationEnabled, "blocks-storage.tsdb.isolation-enabled", true, "Enables TSDB isolation feature. Disabling may improve performance.") + f.BoolVar(&cfg.IsolationEnabled, "blocks-storage.tsdb.isolation-enabled", false, "[Deprecated] Enables TSDB isolation feature. Disabling may improve performance.") } // Validate the config.