From 2314d3453d72eb8e790294ee8b984e0f13d5ea61 Mon Sep 17 00:00:00 2001 From: Charlie Le Date: Fri, 17 May 2024 11:25:23 -0700 Subject: [PATCH] Ingester: Add `upload_compacted_blocks_enabled` config to ingester to parameterize uploading compacted blocks In v1.15.2, ingesters configured with OOO samples ingestion enabled could hit this bug (https://github.com/cortexproject/cortex/issues/5402) where ingesters would not upload compacted blocks (https://github.com/thanos-io/thanos/issues/6462). In v1.16.1, ingesters are configured to always upload compacted blocks (https://github.com/cortexproject/cortex/pull/5625). In v1.17, ingesters stopped uploading compacted blocks (https://github.com/cortexproject/cortex/pull/5735). This can cause problems for users upgrading from v1.15.2 with OOO ingestion enabled to v1.17 because both versions are hard coded to disable uploading compacted blocks from the ingesters. The workaround was to downgrade from v1.17 to v1.16 to allow those compacted blocks to be uploaded (and eventually deleted). The new flag is set to true by default which reverts the behavior of the ingester uploading compacted blocks back to v1.16. Signed-off-by: Charlie Le --- CHANGELOG.md | 1 + docs/configuration/config-file-reference.md | 4 ++++ pkg/ingester/ingester.go | 8 +++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b441f99460..7a55848782 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * [ENHANCEMENT] Distributor/Ring: Allow disabling detailed ring metrics by ring member. #5931 * [ENHANCEMENT] KV: Etcd Added etcd.ping-without-stream-allowed parameter to disable/enable PermitWithoutStream #5933 * [ENHANCEMENT] Ingester: Add a new `max_series_per_label_set` limit. This limit functions similarly to `max_series_per_metric`, but allowing users to define the maximum number of series per LabelSet. #5950 +* [ENHANCEMENT] Ingester: Added `upload_compacted_blocks_enabled` config to ingester to parameterize uploading compacted blocks. * [CHANGE] Upgrade Dockerfile Node version from 14x to 18x. #5906 * [CHANGE] Query Frontend/Ruler: Omit empty data field in API response. #5953 #5954 * [BUGFIX] Configsdb: Fix endline issue in db password. #5920 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 4f330b8738..9920dc4d2c 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2928,6 +2928,10 @@ lifecycler: # CLI flag: -ingester.active-series-metrics-idle-timeout [active_series_metrics_idle_timeout: | default = 10m] +# Enable uploading compacted blocks. +# CLI flag: -ingester.upload-compacted-blocks-enabled +[upload_compacted_blocks_enabled: | default = true] + instance_limits: # Max ingestion rate (samples/sec) that ingester will accept. This limit is # per-ingester, not per-tenant. Additional push requests will be rejected. diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index f739c3fbc4..a8b188428a 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -111,6 +111,9 @@ type Config struct { // Use blocks storage. BlocksStorageConfig cortex_tsdb.BlocksStorageConfig `yaml:"-"` + // UploadCompactedBlocksEnabled enables uploading compacted blocks. + UploadCompactedBlocksEnabled bool `yaml:"upload_compacted_blocks_enabled"` + // Injected at runtime and read from the distributor config, required // to accurately apply global limits. DistributorShardingStrategy string `yaml:"-"` @@ -144,6 +147,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.ActiveSeriesMetricsUpdatePeriod, "ingester.active-series-metrics-update-period", 1*time.Minute, "How often to update active series metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsIdleTimeout, "ingester.active-series-metrics-idle-timeout", 10*time.Minute, "After what time a series is considered to be inactive.") + f.BoolVar(&cfg.UploadCompactedBlocksEnabled, "ingester.upload-compacted-blocks-enabled", true, "Enable uploading compacted blocks.") f.Float64Var(&cfg.DefaultLimits.MaxIngestionRate, "ingester.instance-limits.max-ingestion-rate", 0, "Max ingestion rate (samples/sec) that ingester will accept. This limit is per-ingester, not per-tenant. Additional push requests will be rejected. Current ingestion rate is computed as exponentially weighted moving average, updated every second. This limit only works when using blocks engine. 0 = unlimited.") f.Int64Var(&cfg.DefaultLimits.MaxInMemoryTenants, "ingester.instance-limits.max-tenants", 0, "Max users that this ingester can hold. Requests from additional users will be rejected. This limit only works when using blocks engine. 0 = unlimited.") f.Int64Var(&cfg.DefaultLimits.MaxInMemorySeries, "ingester.instance-limits.max-series", 0, "Max series that this ingester can hold (across all tenants). Requests to create additional series will be rejected. This limit only works when using blocks engine. 0 = unlimited.") @@ -2138,9 +2142,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { func() labels.Labels { return l }, metadata.ReceiveSource, func() bool { - // There is no need to upload compacted blocks since OOO blocks - // won't be compacted due to overlap. - return false + return i.cfg.UploadCompactedBlocksEnabled }, true, // Allow out of order uploads. It's fine in Cortex's context. metadata.NoneFunc,