Update thanos to latest main (#5580)

* update thanos to latest main

Signed-off-by: Ben Ye <[email protected]>

* update changelog

Signed-off-by: Ben Ye <[email protected]>

---------

Signed-off-by: Ben Ye <[email protected]>

Author: yeya24

CHANGELOG.md

@@ -36,7 +36,8 @@
 * [FEATURE] Ruler: Support for filtering rules in the API. #5417
 * [FEATURE] Compactor: Add `-compactor.ring.tokens-file-path` to store generated tokens locally. #5432
 * [FEATURE] Query Frontend: Add `-frontend.retry-on-too-many-outstanding-requests` to re-enqueue 429 requests if there are multiple query-schedulers available. #5496
-* [FEATURE] Store Gateway: Add `-blocks-storage.bucket-store.max-inflight-requests`for store gateways to reject further requests upon reaching the limit. #5553
+* [FEATURE] Store Gateway: Add `-blocks-storage.bucket-store.max-inflight-requests` for store gateways to reject further requests upon reaching the limit. #5553
+* [FEATURE] Store Gateway: Add `cortex_bucket_store_block_load_duration_seconds` histogram to track time to load blocks. #5580
 * [ENHANCEMENT] Distributor/Ingester: Add span on push path #5319
 * [ENHANCEMENT] Support object storage backends for runtime configuration file. #5292
 * [ENHANCEMENT] Query Frontend: Reject subquery with too small step size. #5323

go.mod

@@ -53,7 +53,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed
 	github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e
-	github.com/thanos-io/thanos v0.32.4-0.20230921182036-6257767ec9d0
+	github.com/thanos-io/thanos v0.32.4-0.20230926060504-20d29008068f
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d
 	go.etcd.io/etcd/api/v3 v3.5.9

go.sum

@@ -1212,8 +1212,8 @@ github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed h1:iWQdY3S6DpWj
 github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE=
 github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e h1:kwsFCU8eSkZehbrAN3nXPw5RdMHi/Bok/y8l2C4M+gk=
 github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e/go.mod h1:+T/ZYNCGybT6eTsGGvVtGb63nT1cvUmH6MjqRrcQoKw=
-github.com/thanos-io/thanos v0.32.4-0.20230921182036-6257767ec9d0 h1:T9Vot+BQao6M6j8F0JQbseAqtniOw1Csz+QHRRRwF48=
-github.com/thanos-io/thanos v0.32.4-0.20230921182036-6257767ec9d0/go.mod h1:Px5Boq60s+2WwR+V4v4oxgmxfw9WHrwMwjRou6pkUNw=
+github.com/thanos-io/thanos v0.32.4-0.20230926060504-20d29008068f h1:OdZZLgF2eYIiad7h4WeUPkew7Uq6F9vFPg3aDZfMQLY=
+github.com/thanos-io/thanos v0.32.4-0.20230926060504-20d29008068f/go.mod h1:Px5Boq60s+2WwR+V4v4oxgmxfw9WHrwMwjRou6pkUNw=
 github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU=
 github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

pkg/storegateway/bucket_store_metrics.go

@@ -16,6 +16,7 @@ type BucketStoreMetrics struct {
 	blockLoadFailures     *prometheus.Desc
 	blockDrops            *prometheus.Desc
 	blockDropFailures     *prometheus.Desc
+	blockLoadDuration     *prometheus.Desc
 	blocksLoaded          *prometheus.Desc
 	seriesDataTouched     *prometheus.Desc
 	seriesDataFetched     *prometheus.Desc
@@ -75,6 +76,10 @@ func NewBucketStoreMetrics() *BucketStoreMetrics {
 			"cortex_bucket_store_block_drop_failures_total",
 			"Total number of local blocks that failed to be dropped.",
 			nil, nil),
+		blockLoadDuration: prometheus.NewDesc(
+			"cortex_bucket_store_block_load_duration_seconds",
+			"The total time taken to load a block in seconds.",
+			nil, nil),
 		blocksLoaded: prometheus.NewDesc(
 			"cortex_bucket_store_blocks_loaded",
 			"Number of currently loaded blocks.",
@@ -228,6 +233,7 @@ func (m *BucketStoreMetrics) Describe(out chan<- *prometheus.Desc) {
 	out <- m.blockLoadFailures
 	out <- m.blockDrops
 	out <- m.blockDropFailures
+	out <- m.blockLoadDuration
 	out <- m.blocksLoaded
 	out <- m.seriesDataTouched
 	out <- m.seriesDataFetched
@@ -274,6 +280,7 @@ func (m *BucketStoreMetrics) Collect(out chan<- prometheus.Metric) {
 	data.SendSumOfCounters(out, m.blockLoadFailures, "thanos_bucket_store_block_load_failures_total")
 	data.SendSumOfCounters(out, m.blockDrops, "thanos_bucket_store_block_drops_total")
 	data.SendSumOfCounters(out, m.blockDropFailures, "thanos_bucket_store_block_drop_failures_total")
+	data.SendSumOfHistograms(out, m.blockLoadDuration, "thanos_bucket_store_block_load_duration_seconds")
 
 	data.SendSumOfGaugesPerUser(out, m.blocksLoaded, "thanos_bucket_store_blocks_loaded")
 

pkg/storegateway/bucket_store_metrics_test.go

@@ -41,6 +41,19 @@ func TestBucketStoreMetrics(t *testing.T) {
 			# HELP cortex_bucket_store_block_drops_total Total number of local blocks that were dropped.
 			# TYPE cortex_bucket_store_block_drops_total counter
 			cortex_bucket_store_block_drops_total 90076
+        	# HELP cortex_bucket_store_block_load_duration_seconds The total time taken to load a block in seconds.
+        	# TYPE cortex_bucket_store_block_load_duration_seconds histogram
+        	cortex_bucket_store_block_load_duration_seconds_bucket{le="0.1"} 0
+        	cortex_bucket_store_block_load_duration_seconds_bucket{le="0.5"} 0
+        	cortex_bucket_store_block_load_duration_seconds_bucket{le="1"} 0
+        	cortex_bucket_store_block_load_duration_seconds_bucket{le="10"} 0
+        	cortex_bucket_store_block_load_duration_seconds_bucket{le="20"} 0
+        	cortex_bucket_store_block_load_duration_seconds_bucket{le="30"} 0
+        	cortex_bucket_store_block_load_duration_seconds_bucket{le="60"} 0
+        	cortex_bucket_store_block_load_duration_seconds_bucket{le="120"} 0
+        	cortex_bucket_store_block_load_duration_seconds_bucket{le="+Inf"} 3
+        	cortex_bucket_store_block_load_duration_seconds_sum 112595
+        	cortex_bucket_store_block_load_duration_seconds_count 3
 
 			# HELP cortex_bucket_store_block_drop_failures_total Total number of local blocks that failed to be dropped.
 			# TYPE cortex_bucket_store_block_drop_failures_total counter
@@ -601,6 +614,7 @@ func populateMockedBucketStoreMetrics(base float64) *prometheus.Registry {
 	m.blockLoadFailures.Add(3 * base)
 	m.blockDrops.Add(4 * base)
 	m.blockDropFailures.Add(5 * base)
+	m.blockLoadDuration.Observe(5 * base)
 	m.seriesDataTouched.WithLabelValues("touched-a").Observe(6 * base)
 	m.seriesDataTouched.WithLabelValues("touched-b").Observe(7 * base)
 	m.seriesDataTouched.WithLabelValues("touched-c").Observe(8 * base)
@@ -684,6 +698,7 @@ type mockedBucketStoreMetrics struct {
 	blockLoadFailures     prometheus.Counter
 	blockDrops            prometheus.Counter
 	blockDropFailures     prometheus.Counter
+	blockLoadDuration     prometheus.Histogram
 	seriesDataTouched     *prometheus.HistogramVec
 	seriesDataFetched     *prometheus.HistogramVec
 	seriesDataSizeTouched *prometheus.HistogramVec
@@ -741,6 +756,11 @@ func newMockedBucketStoreMetrics(reg prometheus.Registerer) *mockedBucketStoreMe
 		Name: "thanos_bucket_store_block_drop_failures_total",
 		Help: "Total number of local blocks that failed to be dropped.",
 	})
+	m.blockLoadDuration = promauto.With(reg).NewHistogram(prometheus.HistogramOpts{
+		Name:    "thanos_bucket_store_block_load_duration_seconds",
+		Help:    "The total time taken to load a block in seconds.",
+		Buckets: []float64{0.1, 0.5, 1, 10, 20, 30, 60, 120},
+	})
 	m.blocksLoaded = promauto.With(reg).NewGauge(prometheus.GaugeOpts{
 		Name: "thanos_bucket_store_blocks_loaded",
 		Help: "Number of currently loaded blocks.",