diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cf3e1ed8d9..c34d28f53e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526 * [FEATURE] Update prometheus alertmanager version to v0.28.0 and add new integration msteamsv2, jira, and rocketchat. #6590 -* [FEATURE] Ingester: Add a `-ingester.enable-ooo-native-histograms` flag to enable out-of-order native histogram ingestion per tenant. It only takes effect when `-blocks-storage.tsdb.enable-native-histograms=true` and `-ingester.out-of-order-time-window` > 0. It is applied after the restart if it is changed at runtime through the runtime config. #6626 +* [FEATURE] Ingester: Support out-of-order native histogram ingestion. It automatically enabled when `-ingester.out-of-order-time-window > 0` and `-blocks-storage.tsdb.enable-native-histograms=true`. #6626 #6663 * [ENHANCEMENT] Querier: limit label APIs to query only ingesters if `start` param is not been specified. #6618 * [ENHANCEMENT] Alertmanager: Add new limits `-alertmanager.max-silences-count` and `-alertmanager.max-silences-size-bytes` for limiting silences per tenant. #6605 * [ENHANCEMENT] Update prometheus version to v3.1.0. #6583 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 1b9a03b47b4..0fd956f0288 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3526,13 +3526,6 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -ingester.max-exemplars [max_exemplars: | default = 0] -# [Experimental] Enable out-of-order native histogram ingestion, it only takes -# effect when -blocks-storage.tsdb.enable-native-histograms=true and -# -ingester.out-of-order-time-window > 0. It is applied after the restart if it -# is changed at runtime through the runtime config. -# CLI flag: -ingester.enable-ooo-native-histograms -[enable_ooo_native_histograms: | default = false] - # Maximum number of chunks that can be fetched in a single query from ingesters # and long-term storage. This limit is enforced in the querier, ruler and # store-gateway. 0 to disable. diff --git a/docs/configuration/v1-guarantees.md b/docs/configuration/v1-guarantees.md index d0eeafce433..bf3beed915e 100644 --- a/docs/configuration/v1-guarantees.md +++ b/docs/configuration/v1-guarantees.md @@ -104,7 +104,6 @@ Currently experimental features are: - `-blocks-storage.tsdb.out-of-order-cap-max` (int) CLI flag - `-ingester.out-of-order-time-window` (duration) CLI flag - `out_of_order_time_window` (duration) field in runtime config file - - `enable_ooo_native_histograms` (bool) field in runtime config file - Store Gateway Zone Stable Shuffle Sharding - `-store-gateway.sharding-ring.zone-stable-shuffle-sharding` CLI flag - `zone_stable_shuffle_sharding` (boolean) field in config file diff --git a/docs/guides/native-histograms.md b/docs/guides/native-histograms.md index acbdd40a0cb..da53d685bf7 100644 --- a/docs/guides/native-histograms.md +++ b/docs/guides/native-histograms.md @@ -55,35 +55,14 @@ overrides: ## How to enable out-of-order native histograms ingestion Like samples out-of-order ingestion, the Cortex allows out-of-order ingestion for the native histogram. -To enable it, set the flag `-ingester.enable-ooo-native-histograms`. +It is automatically enabled when `-blocks-storage.tsdb.enable-native-histograms=true` and `-ingester.out-of-order-time-window > 0`. And via yaml: + ```yaml +blocks_storage: + tsdb: + enable_native_histograms: true limits: - enable_ooo_native_histograms: -``` - -Is it only works if when `-blocks-storage.tsdb.enable-native-histograms=true` and `-ingester.out-of-order-time-window > 0`. - -To enable it per tenant, you can utilize a [runtime config](../configuration/arguments.md#runtime-configuration-file). - -For example, the following yaml file specifies enabling out-of-order native histogram ingestion for `user-1`, but not for `user-2`. - -``` -overrides: - user-1: - enable_ooo_native_histograms: true - user-2: - enable_ooo_native_histograms: false -``` - -**Caution**: It is applied after the Ingester restart if it is changed at runtime through the runtime config. -For example, if you have changed the `enable_ooo_native_histograms` value to `false` of the `user-1` via the below yaml file, then the Ingester stops the out-of-order ingestion not until the Ingester restarts. - -``` -overrides: - user-1: - enable_ooo_native_histograms: false - user-2: - enable_ooo_native_histograms: false + out_of_order_time_window: 5m ``` \ No newline at end of file diff --git a/integration/native_histogram_test.go b/integration/native_histogram_test.go index 7dc758a9fb5..68a7c2deb67 100644 --- a/integration/native_histogram_test.go +++ b/integration/native_histogram_test.go @@ -33,7 +33,6 @@ func TestOOONativeHistogramIngestion(t *testing.T) { flags := mergeFlags(baseFlags, map[string]string{ // ooo setting - "-ingester.enable-ooo-native-histograms": "true", "-blocks-storage.tsdb.enable-native-histograms": "true", "-ingester.out-of-order-time-window": "5m", // alert manager diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index ea8ca3c6f12..86a3a3dfdd6 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -2424,8 +2424,8 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { EnableMemorySnapshotOnShutdown: i.cfg.BlocksStorageConfig.TSDB.MemorySnapshotOnShutdown, OutOfOrderTimeWindow: time.Duration(oooTimeWindow).Milliseconds(), OutOfOrderCapMax: i.cfg.BlocksStorageConfig.TSDB.OutOfOrderCapMax, - EnableOOONativeHistograms: i.limits.EnableOOONativeHistograms(userID), - EnableOverlappingCompaction: false, // Always let compactors handle overlapped blocks, e.g. OOO blocks. + EnableOOONativeHistograms: i.cfg.BlocksStorageConfig.TSDB.EnableNativeHistograms, // Automatically enabled when EnableNativeHistograms is true. + EnableOverlappingCompaction: false, // Always let compactors handle overlapped blocks, e.g. OOO blocks. EnableNativeHistograms: i.cfg.BlocksStorageConfig.TSDB.EnableNativeHistograms, BlockChunkQuerierFunc: i.blockChunkQuerierFunc(userID), }, nil) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 65587587a69..c22668b828f 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -1028,7 +1028,6 @@ func TestIngester_Push(t *testing.T) { maxExemplars int oooTimeWindow time.Duration disableNativeHistogram bool - enableOOONativeHistograms bool }{ "should record native histogram discarded": { reqs: []*cortexpb.WriteRequest{ @@ -1517,65 +1516,6 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_active_series{user="test"} 1 `, }, - "native histogram ooo disabled, should soft fail": { - reqs: []*cortexpb.WriteRequest{ - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - nil, - nil, - []cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969, tsdbutil.GenerateTestHistogram(1))}, - cortexpb.API), - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - nil, - nil, - []cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969-(10), tsdbutil.GenerateTestHistogram(1))}, - cortexpb.API), - }, - oooTimeWindow: 5 * time.Minute, - enableOOONativeHistograms: false, - expectedErr: httpgrpc.Errorf(http.StatusBadRequest, "%s", wrapWithUser(wrappedTSDBIngestErr(storage.ErrOOONativeHistogramsDisabled, model.Time(1575043969-(10)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), - expectedIngested: []cortexpb.TimeSeries{ - {Labels: metricLabelAdapters, Histograms: []cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969, tsdbutil.GenerateTestHistogram(1))}}, - }, - additionalMetrics: []string{ - "cortex_ingester_tsdb_head_samples_appended_total", - "cortex_ingester_active_series", - }, - expectedMetrics: ` - # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. - # TYPE cortex_ingester_ingested_samples_failures_total counter - cortex_ingester_ingested_samples_failures_total 0 - # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. - # TYPE cortex_ingester_ingested_samples_total counter - cortex_ingester_ingested_samples_total 0 - # HELP cortex_ingester_ingested_native_histograms_total The total number of native histograms ingested. - # TYPE cortex_ingester_ingested_native_histograms_total counter - cortex_ingester_ingested_native_histograms_total 1 - # HELP cortex_ingester_ingested_native_histograms_failures_total The total number of native histograms that errored on ingestion. - # TYPE cortex_ingester_ingested_native_histograms_failures_total counter - cortex_ingester_ingested_native_histograms_failures_total 1 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - # HELP cortex_ingester_tsdb_head_samples_appended_total Total number of appended samples. - # TYPE cortex_ingester_tsdb_head_samples_appended_total counter - cortex_ingester_tsdb_head_samples_appended_total{type="float",user="test"} 0 - cortex_ingester_tsdb_head_samples_appended_total{type="histogram",user="test"} 1 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 1 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="test"} 1 - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="test"} 0 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test"} 1 - `, - }, "native histogram ooo enabled, should soft fail on sample too old": { reqs: []*cortexpb.WriteRequest{ cortexpb.ToWriteRequest( @@ -1591,9 +1531,8 @@ func TestIngester_Push(t *testing.T) { []cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969-(600*1000), tsdbutil.GenerateTestHistogram(1))}, cortexpb.API), }, - oooTimeWindow: 5 * time.Minute, - enableOOONativeHistograms: true, - expectedErr: httpgrpc.Errorf(http.StatusBadRequest, "%s", wrapWithUser(wrappedTSDBIngestErr(storage.ErrTooOldSample, model.Time(1575043969-(600*1000)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), + oooTimeWindow: 5 * time.Minute, + expectedErr: httpgrpc.Errorf(http.StatusBadRequest, "%s", wrapWithUser(wrappedTSDBIngestErr(storage.ErrTooOldSample, model.Time(1575043969-(600*1000)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), expectedIngested: []cortexpb.TimeSeries{ {Labels: metricLabelAdapters, Histograms: []cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969, tsdbutil.GenerateTestHistogram(1))}}, }, @@ -1654,8 +1593,7 @@ func TestIngester_Push(t *testing.T) { []cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969-(10), tsdbutil.GenerateTestHistogram(1))}, cortexpb.API), }, - oooTimeWindow: 5 * time.Minute, - enableOOONativeHistograms: true, + oooTimeWindow: 5 * time.Minute, expectedIngested: []cortexpb.TimeSeries{ {Labels: metricLabelAdapters, Histograms: []cortexpb.Histogram{cortexpb.HistogramToHistogramProto(1575043969-(10), tsdbutil.GenerateTestHistogram(1)), cortexpb.HistogramToHistogramProto(1575043969, tsdbutil.GenerateTestHistogram(1))}}, }, @@ -2005,7 +1943,6 @@ func TestIngester_Push(t *testing.T) { limits := defaultLimitsTestConfig() limits.MaxExemplars = testData.maxExemplars limits.OutOfOrderTimeWindow = model.Duration(testData.oooTimeWindow) - limits.EnableOOONativeHistograms = testData.enableOOONativeHistograms limits.LimitsPerLabelSet = []validation.LimitsPerLabelSet{ { LabelSet: labels.FromMap(map[string]string{model.MetricNameLabel: "test"}), diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 9a71f24cf54..b39a71c2b0a 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -159,8 +159,6 @@ type Limits struct { OutOfOrderTimeWindow model.Duration `yaml:"out_of_order_time_window" json:"out_of_order_time_window"` // Exemplars MaxExemplars int `yaml:"max_exemplars" json:"max_exemplars"` - // Native Histogram - EnableOOONativeHistograms bool `yaml:"enable_ooo_native_histograms" json:"enable_ooo_native_histograms"` // Querier enforced limits. MaxChunksPerQuery int `yaml:"max_fetched_chunks_per_query" json:"max_fetched_chunks_per_query"` @@ -260,7 +258,6 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxGlobalSeriesPerMetric, "ingester.max-global-series-per-metric", 0, "The maximum number of active series per metric name, across the cluster before replication. 0 to disable.") f.IntVar(&l.MaxExemplars, "ingester.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to blocks-storage.tsdb.max-exemplars value.") f.Var(&l.OutOfOrderTimeWindow, "ingester.out-of-order-time-window", "[Experimental] Configures the allowed time window for ingestion of out-of-order samples. Disabled (0s) by default.") - f.BoolVar(&l.EnableOOONativeHistograms, "ingester.enable-ooo-native-histograms", false, "[Experimental] Enable out-of-order native histogram ingestion, it only takes effect when -blocks-storage.tsdb.enable-native-histograms=true and -ingester.out-of-order-time-window > 0. It is applied after the restart if it is changed at runtime through the runtime config.") f.IntVar(&l.MaxLocalMetricsWithMetadataPerUser, "ingester.max-metadata-per-user", 8000, "The maximum number of active metrics with metadata per user, per ingester. 0 to disable.") f.IntVar(&l.MaxLocalMetadataPerMetric, "ingester.max-metadata-per-metric", 10, "The maximum number of metadata per metric, per ingester. 0 to disable.") @@ -906,11 +903,6 @@ func (o *Overrides) MaxExemplars(userID string) int { return o.GetOverridesForUser(userID).MaxExemplars } -// EnableOOONativeHistograms returns whether to ingest out-of-order native histogram for a given user. -func (o *Overrides) EnableOOONativeHistograms(userID string) bool { - return o.GetOverridesForUser(userID).EnableOOONativeHistograms -} - // Notification limits are special. Limits are returned in following order: // 1. per-tenant limits for given integration // 2. default limits for given integration diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index 79bee8adeaa..414cb3e8d45 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -606,38 +606,6 @@ tenant2: require.Equal(t, 5, ov.MaxExemplars("tenant3")) } -func TestEnableOOONativeHistograms(t *testing.T) { - SetDefaultLimitsForYAMLUnmarshalling(Limits{ - MaxLabelNameLength: 100, - }) - - baseYAML := ` -enable_ooo_native_histograms: false` - overridesYAML := ` -tenant1: - enable_ooo_native_histograms: true -tenant2: - enable_ooo_native_histograms: false -` - - l := Limits{} - err := yaml.UnmarshalStrict([]byte(baseYAML), &l) - require.NoError(t, err) - - overrides := map[string]*Limits{} - err = yaml.Unmarshal([]byte(overridesYAML), &overrides) - require.NoError(t, err, "parsing overrides") - - tl := newMockTenantLimits(overrides) - - ov, err := NewOverrides(l, tl) - require.NoError(t, err) - - require.Equal(t, true, ov.EnableOOONativeHistograms("tenant1")) - require.Equal(t, false, ov.EnableOOONativeHistograms("tenant2")) - require.Equal(t, false, ov.EnableOOONativeHistograms("tenant3")) -} - func TestMaxDownloadedBytesPerRequestOverridesPerTenant(t *testing.T) { SetDefaultLimitsForYAMLUnmarshalling(Limits{ MaxLabelNameLength: 100,