cortexproject · yeya24 · Mar 25, 2025 · Mar 25, 2025
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,7 +5,7 @@
 * [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458
 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526
 * [FEATURE] Update prometheus alertmanager version to v0.28.0 and add new integration msteamsv2, jira, and rocketchat. #6590
-* [FEATURE] Ingester: Add a `-ingester.enable-ooo-native-histograms` flag to enable out-of-order native histogram ingestion per tenant. It only takes effect when `-blocks-storage.tsdb.enable-native-histograms=true` and `-ingester.out-of-order-time-window` > 0. It is applied after the restart if it is changed at runtime through the runtime config. #6626
+* [FEATURE] Ingester: Support out-of-order native histogram ingestion. It automatically enabled when `-ingester.out-of-order-time-window > 0` and `-blocks-storage.tsdb.enable-native-histograms=true`. #6626 #6663
 * [ENHANCEMENT] Querier: limit label APIs to query only ingesters if `start` param is not been specified. #6618
 * [ENHANCEMENT] Alertmanager: Add new limits `-alertmanager.max-silences-count` and `-alertmanager.max-silences-size-bytes` for limiting silences per tenant. #6605
 * [ENHANCEMENT] Update prometheus version to v3.1.0. #6583

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
@@ -3526,13 +3526,6 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 # CLI flag: -ingester.max-exemplars
 [max_exemplars: <int> | default = 0]
 
-# [Experimental] Enable out-of-order native histogram ingestion, it only takes
-# effect when -blocks-storage.tsdb.enable-native-histograms=true and
-# -ingester.out-of-order-time-window > 0. It is applied after the restart if it
-# is changed at runtime through the runtime config.
-# CLI flag: -ingester.enable-ooo-native-histograms
-[enable_ooo_native_histograms: <boolean> | default = false]
-
 # Maximum number of chunks that can be fetched in a single query from ingesters
 # and long-term storage. This limit is enforced in the querier, ruler and
 # store-gateway. 0 to disable.

diff --git a/docs/configuration/v1-guarantees.md b/docs/configuration/v1-guarantees.md
@@ -104,7 +104,6 @@ Currently experimental features are:
   - `-blocks-storage.tsdb.out-of-order-cap-max` (int) CLI flag
   - `-ingester.out-of-order-time-window` (duration) CLI flag
   - `out_of_order_time_window` (duration) field in runtime config file
-  - `enable_ooo_native_histograms` (bool) field in runtime config file
 - Store Gateway Zone Stable Shuffle Sharding
   - `-store-gateway.sharding-ring.zone-stable-shuffle-sharding` CLI flag
   - `zone_stable_shuffle_sharding` (boolean) field in config file

diff --git a/docs/guides/native-histograms.md b/docs/guides/native-histograms.md
@@ -55,35 +55,14 @@ overrides:
 
 ## How to enable out-of-order native histograms ingestion
 Like samples out-of-order ingestion, the Cortex allows out-of-order ingestion for the native histogram.
-To enable it, set the flag `-ingester.enable-ooo-native-histograms`.
+It is automatically enabled when `-blocks-storage.tsdb.enable-native-histograms=true` and `-ingester.out-of-order-time-window > 0`.
 
 And via yaml:
+
 ```yaml
+blocks_storage:
+  tsdb:
+    enable_native_histograms: true
 limits:
-  enable_ooo_native_histograms: <bool>
-```
-
-Is it only works if when `-blocks-storage.tsdb.enable-native-histograms=true` and `-ingester.out-of-order-time-window > 0`.
-
-To enable it per tenant, you can utilize a [runtime config](../configuration/arguments.md#runtime-configuration-file).
-
-For example, the following yaml file specifies enabling out-of-order native histogram ingestion for `user-1`, but not for `user-2`.
-
-```
-overrides:
-  user-1:
-    enable_ooo_native_histograms: true
-  user-2:
-    enable_ooo_native_histograms: false
-```
-
-**Caution**: It is applied after the Ingester restart if it is changed at runtime through the runtime config.
-For example, if you have changed the `enable_ooo_native_histograms` value to `false` of the `user-1` via the below yaml file, then the Ingester stops the out-of-order ingestion not until the Ingester restarts.
-
-```
-overrides:
-  user-1:
-    enable_ooo_native_histograms: false
-  user-2:
-    enable_ooo_native_histograms: false
+  out_of_order_time_window: 5m
 ```
diff --git a/integration/native_histogram_test.go b/integration/native_histogram_test.go
@@ -33,7 +33,6 @@ func TestOOONativeHistogramIngestion(t *testing.T) {
 
 	flags := mergeFlags(baseFlags, map[string]string{
 		// ooo setting
-		"-ingester.enable-ooo-native-histograms":        "true",
 		"-blocks-storage.tsdb.enable-native-histograms": "true",
 		"-ingester.out-of-order-time-window":            "5m",
 		// alert manager

diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go
@@ -2424,8 +2424,8 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) {
 		EnableMemorySnapshotOnShutdown: i.cfg.BlocksStorageConfig.TSDB.MemorySnapshotOnShutdown,
 		OutOfOrderTimeWindow:           time.Duration(oooTimeWindow).Milliseconds(),
 		OutOfOrderCapMax:               i.cfg.BlocksStorageConfig.TSDB.OutOfOrderCapMax,
-		EnableOOONativeHistograms:      i.limits.EnableOOONativeHistograms(userID),
-		EnableOverlappingCompaction:    false, // Always let compactors handle overlapped blocks, e.g. OOO blocks.
+		EnableOOONativeHistograms:      i.cfg.BlocksStorageConfig.TSDB.EnableNativeHistograms, // Automatically enabled when EnableNativeHistograms is true.
+		EnableOverlappingCompaction:    false,                                                 // Always let compactors handle overlapped blocks, e.g. OOO blocks.
 		EnableNativeHistograms:         i.cfg.BlocksStorageConfig.TSDB.EnableNativeHistograms,
 		BlockChunkQuerierFunc:          i.blockChunkQuerierFunc(userID),
 	}, nil)

diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go
@@ -1028,7 +1028,6 @@ func TestIngester_Push(t *testing.T) {
 		maxExemplars              int
 		oooTimeWindow             time.Duration
 		disableNativeHistogram    bool
-		enableOOONativeHistograms bool
 	}{
 		"should record native histogram discarded": {
 			reqs: []*cortexpb.WriteRequest{
@@ -1517,65 +1516,6 @@ func TestIngester_Push(t *testing.T) {
 				cortex_ingester_active_series{user="test"} 1
 			`,
 		},
-		"native histogram ooo disabled, should soft fail": {
-			reqs: []*cortexpb.WriteRequest{
-				cortexpb.ToWriteRequest(
-					[]labels.Labels{metricLabels},
-					nil,
-					nil,
-					[]cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969, tsdbutil.GenerateTestHistogram(1))},
-					cortexpb.API),
-				cortexpb.ToWriteRequest(
-					[]labels.Labels{metricLabels},
-					nil,
-					nil,
-					[]cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969-(10), tsdbutil.GenerateTestHistogram(1))},
-					cortexpb.API),
-			},
-			oooTimeWindow:             5 * time.Minute,
-			enableOOONativeHistograms: false,
-			expectedErr:               httpgrpc.Errorf(http.StatusBadRequest, "%s", wrapWithUser(wrappedTSDBIngestErr(storage.ErrOOONativeHistogramsDisabled, model.Time(1575043969-(10)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()),
-			expectedIngested: []cortexpb.TimeSeries{
-				{Labels: metricLabelAdapters, Histograms: []cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969, tsdbutil.GenerateTestHistogram(1))}},
-			},
-			additionalMetrics: []string{
-				"cortex_ingester_tsdb_head_samples_appended_total",
-				"cortex_ingester_active_series",
-			},
-			expectedMetrics: `
-				# HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion.
-				# TYPE cortex_ingester_ingested_samples_failures_total counter
-				cortex_ingester_ingested_samples_failures_total 0
-				# HELP cortex_ingester_ingested_samples_total The total number of samples ingested.
-				# TYPE cortex_ingester_ingested_samples_total counter
-				cortex_ingester_ingested_samples_total 0
-				# HELP cortex_ingester_ingested_native_histograms_total The total number of native histograms ingested.
-				# TYPE cortex_ingester_ingested_native_histograms_total counter
-				cortex_ingester_ingested_native_histograms_total 1
-				# HELP cortex_ingester_ingested_native_histograms_failures_total The total number of native histograms that errored on ingestion.
-				# TYPE cortex_ingester_ingested_native_histograms_failures_total counter
-				cortex_ingester_ingested_native_histograms_failures_total 1
-				# HELP cortex_ingester_memory_users The current number of users in memory.
-				# TYPE cortex_ingester_memory_users gauge
-				cortex_ingester_memory_users 1
-        	    # HELP cortex_ingester_tsdb_head_samples_appended_total Total number of appended samples.
-        	    # TYPE cortex_ingester_tsdb_head_samples_appended_total counter
-        	    cortex_ingester_tsdb_head_samples_appended_total{type="float",user="test"} 0
-        	    cortex_ingester_tsdb_head_samples_appended_total{type="histogram",user="test"} 1
-				# HELP cortex_ingester_memory_series The current number of series in memory.
-				# TYPE cortex_ingester_memory_series gauge
-				cortex_ingester_memory_series 1
-				# HELP cortex_ingester_memory_series_created_total The total number of series that were created per user.
-				# TYPE cortex_ingester_memory_series_created_total counter
-				cortex_ingester_memory_series_created_total{user="test"} 1
-				# HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user.
-				# TYPE cortex_ingester_memory_series_removed_total counter
-				cortex_ingester_memory_series_removed_total{user="test"} 0
-				# HELP cortex_ingester_active_series Number of currently active series per user.
-				# TYPE cortex_ingester_active_series gauge
-				cortex_ingester_active_series{user="test"} 1
-			`,
-		},
 		"native histogram ooo enabled, should soft fail on sample too old": {
 			reqs: []*cortexpb.WriteRequest{
 				cortexpb.ToWriteRequest(
@@ -1591,9 +1531,8 @@ func TestIngester_Push(t *testing.T) {
 					[]cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969-(600*1000), tsdbutil.GenerateTestHistogram(1))},
 					cortexpb.API),
 			},
-			oooTimeWindow:             5 * time.Minute,
-			enableOOONativeHistograms: true,
-			expectedErr:               httpgrpc.Errorf(http.StatusBadRequest, "%s", wrapWithUser(wrappedTSDBIngestErr(storage.ErrTooOldSample, model.Time(1575043969-(600*1000)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()),
+			oooTimeWindow: 5 * time.Minute,
+			expectedErr:   httpgrpc.Errorf(http.StatusBadRequest, "%s", wrapWithUser(wrappedTSDBIngestErr(storage.ErrTooOldSample, model.Time(1575043969-(600*1000)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()),
 			expectedIngested: []cortexpb.TimeSeries{
 				{Labels: metricLabelAdapters, Histograms: []cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969, tsdbutil.GenerateTestHistogram(1))}},
 			},
@@ -1654,8 +1593,7 @@ func TestIngester_Push(t *testing.T) {
 					[]cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969-(10), tsdbutil.GenerateTestHistogram(1))},
 					cortexpb.API),
 			},
-			oooTimeWindow:             5 * time.Minute,
-			enableOOONativeHistograms: true,
+			oooTimeWindow: 5 * time.Minute,
 			expectedIngested: []cortexpb.TimeSeries{
 				{Labels: metricLabelAdapters, Histograms: []cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969-(10), tsdbutil.GenerateTestHistogram(1)), cortexpb.HistogramToHistogramProto(1575043969, tsdbutil.GenerateTestHistogram(1))}},
 			},
@@ -2005,7 +1943,6 @@ func TestIngester_Push(t *testing.T) {
 			limits := defaultLimitsTestConfig()
 			limits.MaxExemplars = testData.maxExemplars
 			limits.OutOfOrderTimeWindow = model.Duration(testData.oooTimeWindow)
-			limits.EnableOOONativeHistograms = testData.enableOOONativeHistograms
 			limits.LimitsPerLabelSet = []validation.LimitsPerLabelSet{
 				{
 					LabelSet: labels.FromMap(map[string]string{model.MetricNameLabel: "test"}),

diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
@@ -159,8 +159,6 @@ type Limits struct {
 	OutOfOrderTimeWindow model.Duration `yaml:"out_of_order_time_window" json:"out_of_order_time_window"`
 	// Exemplars
 	MaxExemplars int `yaml:"max_exemplars" json:"max_exemplars"`
-	// Native Histogram
-	EnableOOONativeHistograms bool `yaml:"enable_ooo_native_histograms" json:"enable_ooo_native_histograms"`
 
 	// Querier enforced limits.
 	MaxChunksPerQuery            int            `yaml:"max_fetched_chunks_per_query" json:"max_fetched_chunks_per_query"`
@@ -260,7 +258,6 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) {
 	f.IntVar(&l.MaxGlobalSeriesPerMetric, "ingester.max-global-series-per-metric", 0, "The maximum number of active series per metric name, across the cluster before replication. 0 to disable.")
 	f.IntVar(&l.MaxExemplars, "ingester.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to blocks-storage.tsdb.max-exemplars value.")
 	f.Var(&l.OutOfOrderTimeWindow, "ingester.out-of-order-time-window", "[Experimental] Configures the allowed time window for ingestion of out-of-order samples. Disabled (0s) by default.")
-	f.BoolVar(&l.EnableOOONativeHistograms, "ingester.enable-ooo-native-histograms", false, "[Experimental] Enable out-of-order native histogram ingestion, it only takes effect when -blocks-storage.tsdb.enable-native-histograms=true and -ingester.out-of-order-time-window > 0. It is applied after the restart if it is changed at runtime through the runtime config.")
 
 	f.IntVar(&l.MaxLocalMetricsWithMetadataPerUser, "ingester.max-metadata-per-user", 8000, "The maximum number of active metrics with metadata per user, per ingester. 0 to disable.")
 	f.IntVar(&l.MaxLocalMetadataPerMetric, "ingester.max-metadata-per-metric", 10, "The maximum number of metadata per metric, per ingester. 0 to disable.")
@@ -906,11 +903,6 @@ func (o *Overrides) MaxExemplars(userID string) int {
 	return o.GetOverridesForUser(userID).MaxExemplars
 }
 
-// EnableOOONativeHistograms returns whether to ingest out-of-order native histogram for a given user.
-func (o *Overrides) EnableOOONativeHistograms(userID string) bool {
-	return o.GetOverridesForUser(userID).EnableOOONativeHistograms
-}
-
 // Notification limits are special. Limits are returned in following order:
 // 1. per-tenant limits for given integration
 // 2. default limits for given integration

diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go
@@ -606,38 +606,6 @@ tenant2:
 	require.Equal(t, 5, ov.MaxExemplars("tenant3"))
 }
 
-func TestEnableOOONativeHistograms(t *testing.T) {
-	SetDefaultLimitsForYAMLUnmarshalling(Limits{
-		MaxLabelNameLength: 100,
-	})
-
-	baseYAML := `
-enable_ooo_native_histograms: false`
-	overridesYAML := `
-tenant1:
-  enable_ooo_native_histograms: true
-tenant2:
-  enable_ooo_native_histograms: false
-`
-
-	l := Limits{}
-	err := yaml.UnmarshalStrict([]byte(baseYAML), &l)
-	require.NoError(t, err)
-
-	overrides := map[string]*Limits{}
-	err = yaml.Unmarshal([]byte(overridesYAML), &overrides)
-	require.NoError(t, err, "parsing overrides")
-
-	tl := newMockTenantLimits(overrides)
-
-	ov, err := NewOverrides(l, tl)
-	require.NoError(t, err)
-
-	require.Equal(t, true, ov.EnableOOONativeHistograms("tenant1"))
-	require.Equal(t, false, ov.EnableOOONativeHistograms("tenant2"))
-	require.Equal(t, false, ov.EnableOOONativeHistograms("tenant3"))
-}
-
 func TestMaxDownloadedBytesPerRequestOverridesPerTenant(t *testing.T) {
 	SetDefaultLimitsForYAMLUnmarshalling(Limits{
 		MaxLabelNameLength: 100,