Add a flag to enable TSDB memorry snapshot on shutdown

Signed-off-by: wangguoliang <[email protected]>

Author: wgliang

CHANGELOG.md

@@ -8,6 +8,7 @@
 * [FEATURE] Ingester: Add active series to all_user_stats page. #4972
 * [FEATURE] Ingester: Added `-blocks-storage.tsdb.head-chunks-write-queue-size` allowing to configure the size of the in-memory queue used before flushing chunks to the disk . #5000
 * [FEATURE] Query Frontend: Log query params in query frontend even if error happens. #5005
+* [FEATURE] Ingester: Enable snapshotting of In-memory TSDB on disk during shutdown via `-blocks-storage.tsdb.memory-snapshot-on-shutdown`. And we can also control the behavior of a per user through `blocks-storage.tsdb.memory-snapshot-on-shutdown`. #5011
 * [BUGFIX] Updated `golang.org/x/net` dependency to fix CVE-2022-27664. #5008
 
 ## 1.14.0 in progress

docs/blocks-storage/querier.md

@@ -882,4 +882,9 @@ blocks_storage:
     # will be stored. 0 or less means disabled.
     # CLI flag: -blocks-storage.tsdb.max-exemplars
     [max_exemplars: <int> | default = 0]
+
+    # True to enable snapshotting of in-memory TSDB data on disk when shutting
+    # down.
+    # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown
+    [memory_snapshot_on_shutdown: <boolean> | default = false]
 ```

docs/blocks-storage/store-gateway.md

@@ -946,4 +946,9 @@ blocks_storage:
     # will be stored. 0 or less means disabled.
     # CLI flag: -blocks-storage.tsdb.max-exemplars
     [max_exemplars: <int> | default = 0]
+
+    # True to enable snapshotting of in-memory TSDB data on disk when shutting
+    # down.
+    # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown
+    [memory_snapshot_on_shutdown: <boolean> | default = false]
 ```

docs/configuration/config-file-reference.md

@@ -2669,6 +2669,11 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 # CLI flag: -ingester.max-global-metadata-per-metric
 [max_global_metadata_per_metric: <int> | default = 0]
 
+# True to enable snapshotting in-memory TSDB data of per user on disk when
+# shutting down.
+# CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown-per-user
+[memory_snapshot_on_shutdown_per_user: <boolean> | default = false]
+
 # Maximum number of chunks that can be fetched in a single query from ingesters
 # and long-term storage. This limit is enforced in the querier, ruler and
 # store-gateway. 0 to disable.
@@ -3725,6 +3730,11 @@ tsdb:
   # be stored. 0 or less means disabled.
   # CLI flag: -blocks-storage.tsdb.max-exemplars
   [max_exemplars: <int> | default = 0]
+
+  # True to enable snapshotting of in-memory TSDB data on disk when shutting
+  # down.
+  # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown
+  [memory_snapshot_on_shutdown: <boolean> | default = false]
 ```
 
 ### `compactor_config`

docs/configuration/v1-guarantees.md

@@ -98,4 +98,7 @@ Currently experimental features are:
   - Enabled via `-compactor.sharding-enabled=true`, `-compactor.sharding-strategy=shuffle-sharding`, and `-compactor.tenant-shard-size` set to a value larger than 0.
 - Vertical sharding at query frontend for range/instant queries
   - `-frontend.query-vertical-shard-size` (int) CLI flag
-  - `query_vertical_shard_size` (int) field in runtime config file
+  - `query_vertical_shard_size` (int) field in runtime config file
+- Snapshotting of in-memory TSDB on disk during shutdown
+  - `-blocks-storage.tsdb.memory-snapshot-on-shutdown` (boolean) CLI flag
+  - And we can also control the behavior of a per user through `blocks-storage.tsdb.memory-snapshot-on-shutdown-per-user`

pkg/ingester/ingester.go

@@ -1833,20 +1833,21 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) {
 	}
 	// Create a new user database
 	db, err := tsdb.Open(udir, userLogger, tsdbPromReg, &tsdb.Options{
-		RetentionDuration:         i.cfg.BlocksStorageConfig.TSDB.Retention.Milliseconds(),
-		MinBlockDuration:          blockRanges[0],
-		MaxBlockDuration:          blockRanges[len(blockRanges)-1],
-		NoLockfile:                true,
-		StripeSize:                i.cfg.BlocksStorageConfig.TSDB.StripeSize,
-		HeadChunksWriteBufferSize: i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteBufferSize,
-		WALCompression:            i.cfg.BlocksStorageConfig.TSDB.WALCompressionEnabled,
-		WALSegmentSize:            i.cfg.BlocksStorageConfig.TSDB.WALSegmentSizeBytes,
-		SeriesLifecycleCallback:   userDB,
-		BlocksToDelete:            userDB.blocksToDelete,
-		EnableExemplarStorage:     enableExemplars,
-		IsolationDisabled:         true,
-		MaxExemplars:              int64(i.cfg.BlocksStorageConfig.TSDB.MaxExemplars),
-		HeadChunksWriteQueueSize:  i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteQueueSize,
+		RetentionDuration:              i.cfg.BlocksStorageConfig.TSDB.Retention.Milliseconds(),
+		MinBlockDuration:               blockRanges[0],
+		MaxBlockDuration:               blockRanges[len(blockRanges)-1],
+		NoLockfile:                     true,
+		StripeSize:                     i.cfg.BlocksStorageConfig.TSDB.StripeSize,
+		HeadChunksWriteBufferSize:      i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteBufferSize,
+		WALCompression:                 i.cfg.BlocksStorageConfig.TSDB.WALCompressionEnabled,
+		WALSegmentSize:                 i.cfg.BlocksStorageConfig.TSDB.WALSegmentSizeBytes,
+		SeriesLifecycleCallback:        userDB,
+		BlocksToDelete:                 userDB.blocksToDelete,
+		EnableExemplarStorage:          enableExemplars,
+		IsolationDisabled:              true,
+		MaxExemplars:                   int64(i.cfg.BlocksStorageConfig.TSDB.MaxExemplars),
+		HeadChunksWriteQueueSize:       i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteQueueSize,
+		EnableMemorySnapshotOnShutdown: i.cfg.BlocksStorageConfig.TSDB.MemorySnapshotOnShutdown,
 	}, nil)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to open TSDB: %s", udir)

pkg/storage/tsdb/config.go

@@ -151,6 +151,9 @@ type TSDBConfig struct {
 
 	// Positive value enables experiemental support for exemplars. 0 or less to disable.
 	MaxExemplars int `yaml:"max_exemplars"`
+
+	// Enable snapshotting of in-memory TSDB data on disk when shutting down.
+	MemorySnapshotOnShutdown bool `yaml:"memory_snapshot_on_shutdown"`
 }
 
 // RegisterFlags registers the TSDBConfig flags.
@@ -176,6 +179,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) {
 	f.DurationVar(&cfg.CloseIdleTSDBTimeout, "blocks-storage.tsdb.close-idle-tsdb-timeout", 0, "If TSDB has not received any data for this duration, and all blocks from TSDB have been shipped, TSDB is closed and deleted from local disk. If set to positive value, this value should be equal or higher than -querier.query-ingesters-within flag to make sure that TSDB is not closed prematurely, which could cause partial query results. 0 or negative value disables closing of idle TSDB.")
 	f.IntVar(&cfg.MaxExemplars, "blocks-storage.tsdb.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. 0 or less means disabled.")
 	f.IntVar(&cfg.HeadChunksWriteQueueSize, "blocks-storage.tsdb.head-chunks-write-queue-size", chunks.DefaultWriteQueueSize, "The size of the in-memory queue used before flushing chunks to the disk.")
+	f.BoolVar(&cfg.MemorySnapshotOnShutdown, "blocks-storage.tsdb.memory-snapshot-on-shutdown", false, "True to enable snapshotting of in-memory TSDB data on disk when shutting down.")
 }
 
 // Validate the config.

pkg/util/validation/limits.go

@@ -68,6 +68,8 @@ type Limits struct {
 	MaxLocalMetadataPerMetric           int `yaml:"max_metadata_per_metric" json:"max_metadata_per_metric"`
 	MaxGlobalMetricsWithMetadataPerUser int `yaml:"max_global_metadata_per_user" json:"max_global_metadata_per_user"`
 	MaxGlobalMetadataPerMetric          int `yaml:"max_global_metadata_per_metric" json:"max_global_metadata_per_metric"`
+	// Snapshot
+	MemorySnapshotOnShutdown bool `yaml:"memory_snapshot_on_shutdown" json:"memory_snapshot_on_shutdown"`
 
 	// Querier enforced limits.
 	MaxChunksPerQuery            int            `yaml:"max_fetched_chunks_per_query" json:"max_fetched_chunks_per_query"`
@@ -147,6 +149,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) {
 	f.IntVar(&l.MaxLocalSeriesPerMetric, "ingester.max-series-per-metric", 50000, "The maximum number of active series per metric name, per ingester. 0 to disable.")
 	f.IntVar(&l.MaxGlobalSeriesPerUser, "ingester.max-global-series-per-user", 0, "The maximum number of active series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.")
 	f.IntVar(&l.MaxGlobalSeriesPerMetric, "ingester.max-global-series-per-metric", 0, "The maximum number of active series per metric name, across the cluster before replication. 0 to disable.")
+	f.BoolVar(&l.MemorySnapshotOnShutdown, "blocks-storage.tsdb.memory-snapshot-on-shutdown", false, "True to enable snapshotting in-memory TSDB data of per user on disk when shutting down.")
 
 	f.IntVar(&l.MaxLocalMetricsWithMetadataPerUser, "ingester.max-metadata-per-user", 8000, "The maximum number of active metrics with metadata per user, per ingester. 0 to disable.")
 	f.IntVar(&l.MaxLocalMetadataPerMetric, "ingester.max-metadata-per-metric", 10, "The maximum number of metadata per metric, per ingester. 0 to disable.")
@@ -391,6 +394,12 @@ func (o *Overrides) MaxGlobalSeriesPerMetric(userID string) int {
 	return o.GetOverridesForUser(userID).MaxGlobalSeriesPerMetric
 }
 
+// MemorySnapshotOnShutdown returns true when we should snapshot in-memory TSDB
+// data of per user on disk when shutting down.
+func (o *Overrides) MemorySnapshotOnShutdown(userID string) bool {
+	return o.GetOverridesForUser(userID).MemorySnapshotOnShutdown
+}
+
 // MaxChunksPerQueryFromStore returns the maximum number of chunks allowed per query when fetching
 // chunks from the long-term storage.
 func (o *Overrides) MaxChunksPerQueryFromStore(userID string) int {