Modify default values for deletion delays. (#2414)

* Modify default values for deletion delays.

Signed-off-by: Peter Štibraný <[email protected]>

* Updated documentation

Signed-off-by: Peter Štibraný <[email protected]>

* Updated CHANGELOG.md with PR number.

Signed-off-by: Peter Štibraný <[email protected]>

Author: pstibrany

CHANGELOG.md

@@ -8,6 +8,7 @@
   * `cortex_querier_bucket_store_blocks_meta_sync_failures_total` > `cortex_querier_blocks_meta_sync_failures_total`
   * `cortex_querier_bucket_store_blocks_meta_sync_duration_seconds` > `cortex_querier_blocks_meta_sync_duration_seconds`
   * `cortex_querier_bucket_store_blocks_meta_sync_consistency_delay_seconds` > `cortex_querier_blocks_meta_sync_consistency_delay_seconds`
+* [CHANGE] Experimental TSDB: Modified default values for `compactor.deletion-delay` option from 48h to 12h and `-experimental.tsdb.bucket-store.ignore-deletion-marks-delay` from 24h to 6h. #2414
 * [ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing `cortex_discarded_samples_total` metric. #2370
 * [ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. #2383 
 * [ENHANCEMENT] Experimental TSDB: Added `cortex_querier_blocks_meta_synced`, which reflects current state of synced blocks over all tenants. #2392

docs/configuration/config-file-reference.md

@@ -2441,10 +2441,10 @@ bucket_store:
   # while fetching blocks. The idea of ignore-deletion-marks-delay is to ignore
   # blocks that are marked for deletion with some delay. This ensures store can
   # still serve blocks that are meant to be deleted but do not have a
-  # replacement yet.Default is 24h, half of the default value for
+  # replacement yet.Default is 6h, half of the default value for
   # -compactor.deletion-delay.
   # CLI flag: -experimental.tsdb.bucket-store.ignore-deletion-marks-delay
-  [ignore_deletion_mark_delay: <duration> | default = 24h0m0s]
+  [ignore_deletion_mark_delay: <duration> | default = 6h0m0s]
 
 # How frequently does Cortex try to compact TSDB head. Block is only created if
 # data covers smallest block range. Must be greater than 0 and max 5 minutes.
@@ -2571,7 +2571,7 @@ The `compactor_config` configures the compactor for the experimental blocks stor
 # failures, if store gateway still has the block loaded, or compactor is
 # ignoring the deletion because it's compacting the block at the same time.
 # CLI flag: -compactor.deletion-delay
-[deletion_delay: <duration> | default = 48h0m0s]
+[deletion_delay: <duration> | default = 12h0m0s]
 
 # Shard tenants across multiple compactor instances. Sharding is required if you
 # run multiple compactor instances, in order to coordinate compactions and avoid

docs/operations/blocks-storage.md

@@ -265,10 +265,10 @@ tsdb:
     # while fetching blocks. The idea of ignore-deletion-marks-delay is to
     # ignore blocks that are marked for deletion with some delay. This ensures
     # store can still serve blocks that are meant to be deleted but do not have
-    # a replacement yet.Default is 24h, half of the default value for
+    # a replacement yet.Default is 6h, half of the default value for
     # -compactor.deletion-delay.
     # CLI flag: -experimental.tsdb.bucket-store.ignore-deletion-marks-delay
-    [ignore_deletion_mark_delay: <duration> | default = 24h0m0s]
+    [ignore_deletion_mark_delay: <duration> | default = 6h0m0s]
 
   # How frequently does Cortex try to compact TSDB head. Block is only created
   # if data covers smallest block range. Must be greater than 0 and max 5
@@ -398,7 +398,7 @@ compactor:
   # compactor is ignoring the deletion because it's compacting the block at the
   # same time.
   # CLI flag: -compactor.deletion-delay
-  [deletion_delay: <duration> | default = 48h0m0s]
+  [deletion_delay: <duration> | default = 12h0m0s]
 
   # Shard tenants across multiple compactor instances. Sharding is required if
   # you run multiple compactor instances, in order to coordinate compactions and

pkg/compactor/compactor.go

@@ -64,7 +64,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.DurationVar(&cfg.CompactionInterval, "compactor.compaction-interval", time.Hour, "The frequency at which the compaction runs")
 	f.IntVar(&cfg.CompactionRetries, "compactor.compaction-retries", 3, "How many times to retry a failed compaction during a single compaction interval")
 	f.BoolVar(&cfg.ShardingEnabled, "compactor.sharding-enabled", false, "Shard tenants across multiple compactor instances. Sharding is required if you run multiple compactor instances, in order to coordinate compactions and avoid race conditions leading to the same tenant blocks simultaneously compacted by different instances.")
-	f.DurationVar(&cfg.DeletionDelay, "compactor.deletion-delay", 48*time.Hour, "Time before a block marked for deletion is deleted from bucket. "+
+	f.DurationVar(&cfg.DeletionDelay, "compactor.deletion-delay", 12*time.Hour, "Time before a block marked for deletion is deleted from bucket. "+
 		"If not 0, blocks will be marked for deletion and compactor component will delete blocks marked for deletion from the bucket. "+
 		"If delete-delay is 0, blocks will be deleted straight away. Note that deleting blocks immediately can cause query failures, "+
 		"if store gateway still has the block loaded, or compactor is ignoring the deletion because it's compacting the block at the same time.")

pkg/storage/tsdb/config.go

@@ -184,9 +184,9 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) {
 	f.IntVar(&cfg.MetaSyncConcurrency, "experimental.tsdb.bucket-store.meta-sync-concurrency", 20, "Number of Go routines to use when syncing block meta files from object storage per tenant.")
 	f.BoolVar(&cfg.BinaryIndexHeader, "experimental.tsdb.bucket-store.binary-index-header-enabled", true, "Whether the bucket store should use the binary index header. If false, it uses the JSON index header.")
 	f.DurationVar(&cfg.ConsistencyDelay, "experimental.tsdb.bucket-store.consistency-delay", 0, "Minimum age of a block before it's being read. Set it to safe value (e.g 30m) if your object storage is eventually consistent. GCS and S3 are (roughly) strongly consistent.")
-	f.DurationVar(&cfg.IgnoreDeletionMarksDelay, "experimental.tsdb.bucket-store.ignore-deletion-marks-delay", time.Hour*24, "Duration after which the blocks marked for deletion will be filtered out while fetching blocks. "+
+	f.DurationVar(&cfg.IgnoreDeletionMarksDelay, "experimental.tsdb.bucket-store.ignore-deletion-marks-delay", time.Hour*6, "Duration after which the blocks marked for deletion will be filtered out while fetching blocks. "+
 		"The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet."+
-		"Default is 24h, half of the default value for -compactor.deletion-delay.")
+		"Default is 6h, half of the default value for -compactor.deletion-delay.")
 }
 
 // Validate the config.