From 8889f1b265c08904fed23eda90980492ec9cda30 Mon Sep 17 00:00:00 2001 From: Anna Tran Date: Wed, 27 Nov 2024 10:56:49 -0800 Subject: [PATCH 1/2] Add label size bytes native histogram Signed-off-by: Anna Tran --- CHANGELOG.md | 1 + pkg/util/validation/validate.go | 12 ++++++++++++ pkg/util/validation/validate_test.go | 8 ++++++++ 3 files changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f8b7abab9a..7534e465d34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ * [ENHANCEMENT] Ingester: Introduce a new experimental feature for caching expanded postings on the ingester. #6296 * [ENHANCEMENT] Querier/Ruler: Expose `store_gateway_consistency_check_max_attempts` for max retries when querying store gateway in consistency check. #6276 * [ENHANCEMENT] StoreGateway: Add new `cortex_bucket_store_chunk_pool_inuse_bytes` metric to track the usage in chunk pool. #6310 +* [ENHANCEMENT] Distributor: Expose `cortex_label_size_bytes` native histogram metric. #6372 * [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224 * [BUGFIX] Ruler: Allow rule evaluation to complete during shutdown. #6326 * [BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled #6271 diff --git a/pkg/util/validation/validate.go b/pkg/util/validation/validate.go index be94cfa2f13..2cb8999a0aa 100644 --- a/pkg/util/validation/validate.go +++ b/pkg/util/validation/validate.go @@ -78,6 +78,7 @@ type ValidateMetrics struct { DiscardedExemplars *prometheus.CounterVec DiscardedMetadata *prometheus.CounterVec HistogramSamplesReducedResolution *prometheus.CounterVec + LabelSizeBytes *prometheus.HistogramVec } func registerCollector(r prometheus.Registerer, c prometheus.Collector) { @@ -120,11 +121,21 @@ func NewValidateMetrics(r prometheus.Registerer) *ValidateMetrics { []string{"user"}, ) registerCollector(r, histogramSamplesReducedResolution) + labelSizeBytes := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "cortex_label_size_bytes", + Help: "The combined size in bytes of all labels and label values for a time series.", + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }, []string{"user"}) + registerCollector(r, labelSizeBytes) + m := &ValidateMetrics{ DiscardedSamples: discardedSamples, DiscardedExemplars: discardedExemplars, DiscardedMetadata: discardedMetadata, HistogramSamplesReducedResolution: histogramSamplesReducedResolution, + LabelSizeBytes: labelSizeBytes, } return m @@ -236,6 +247,7 @@ func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID str lastLabelName = l.Name labelsSizeBytes += l.Size() } + validateMetrics.LabelSizeBytes.WithLabelValues(userID).Observe(float64(labelsSizeBytes)) if maxLabelsSizeBytes > 0 && labelsSizeBytes > maxLabelsSizeBytes { validateMetrics.DiscardedSamples.WithLabelValues(labelsSizeBytesExceeded, userID).Inc() return labelSizeBytesExceededError(ls, labelsSizeBytes, maxLabelsSizeBytes) diff --git a/pkg/util/validation/validate_test.go b/pkg/util/validation/validate_test.go index 7d7cf459300..6b6fb00bbf5 100644 --- a/pkg/util/validation/validate_test.go +++ b/pkg/util/validation/validate_test.go @@ -119,6 +119,14 @@ func TestValidateLabels(t *testing.T) { cortex_discarded_samples_total{reason="random reason",user="different user"} 1 `), "cortex_discarded_samples_total")) + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_label_size_bytes The combined size in bytes of all labels and label values for a time series. + # TYPE cortex_label_size_bytes histogram + cortex_label_size_bytes_bucket{user="testUser",le="+Inf"} 3 + cortex_label_size_bytes_sum{user="testUser"} 148 + cortex_label_size_bytes_count{user="testUser"} 3 + `), "cortex_label_size_bytes")) + DeletePerUserValidationMetrics(validateMetrics, userID, util_log.Logger) require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` From 4c722d87ee86d71fe6fb3608e46f81c69cf0601f Mon Sep 17 00:00:00 2001 From: Anna Tran Date: Wed, 27 Nov 2024 14:01:54 -0800 Subject: [PATCH 2/2] Update DeletePerUserValidationMetrics with labelSizeBytes Signed-off-by: Anna Tran --- pkg/util/validation/validate.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/util/validation/validate.go b/pkg/util/validation/validate.go index 2cb8999a0aa..3e36ddf529d 100644 --- a/pkg/util/validation/validate.go +++ b/pkg/util/validation/validate.go @@ -364,4 +364,7 @@ func DeletePerUserValidationMetrics(validateMetrics *ValidateMetrics, userID str if err := util.DeleteMatchingLabels(validateMetrics.HistogramSamplesReducedResolution, filter); err != nil { level.Warn(log).Log("msg", "failed to remove cortex_reduced_resolution_histogram_samples_total metric for user", "user", userID, "err", err) } + if err := util.DeleteMatchingLabels(validateMetrics.LabelSizeBytes, filter); err != nil { + level.Warn(log).Log("msg", "failed to remove cortex_label_size_bytes metric for user", "user", userID, "err", err) + } }