Skip to content

Commit ba6b14a

Browse files
yeya24yeya24
authored
expose indexheader download and load duration histograms (#6445)
* expose indexheader download and load duration histograms Signed-off-by: Ben Ye <[email protected]> * changelog Signed-off-by: Ben Ye <[email protected]> * changelog Signed-off-by: Ben Ye <[email protected]> --------- Signed-off-by: Ben Ye <[email protected]>
1 parent 7b1ed51 commit ba6b14a

File tree

3 files changed

+61
-0
lines changed

3 files changed

+61
-0
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
* [ENHANCEMENT] Ingester: If a limit per label set entry doesn't have any label, use it as the default partition to catch all series that doesn't match any other label sets entries. #6435
5353
* [ENHANCEMENT] Querier: Add new `cortex_querier_codec_response_size` metric to track the size of the encoded query responses from queriers. #6444
5454
* [ENHANCEMENT] Distributor: Added `cortex_distributor_received_samples_per_labelset_total` metric to calculate ingestion rate per label set. #6443
55+
* [ENHANCEMENT] StoreGateway: Added `cortex_bucket_store_indexheader_load_duration_seconds` and `cortex_bucket_store_indexheader_download_duration_seconds` metrics for time of downloading and loading index header files. #6445
5556
* [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224
5657
* [BUGFIX] Ruler: Allow rule evaluation to complete during shutdown. #6326
5758
* [BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled. #6271

‎pkg/storegateway/bucket_store_metrics.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ type BucketStoreMetrics struct {
5555
indexHeaderLazyUnloadCount *prometheus.Desc
5656
indexHeaderLazyUnloadFailedCount *prometheus.Desc
5757
indexHeaderLazyLoadDuration *prometheus.Desc
58+
indexHeaderDownloadDuration *prometheus.Desc
59+
indexHeaderLoadDuration *prometheus.Desc
5860
}
5961

6062
func NewBucketStoreMetrics() *BucketStoreMetrics {
@@ -205,6 +207,14 @@ func NewBucketStoreMetrics() *BucketStoreMetrics {
205207
"cortex_bucket_store_indexheader_lazy_load_duration_seconds",
206208
"Duration of the index-header lazy loading in seconds.",
207209
nil, nil),
210+
indexHeaderDownloadDuration: prometheus.NewDesc(
211+
"cortex_bucket_store_indexheader_download_duration_seconds",
212+
"Duration of the index-header download from objstore in seconds.",
213+
nil, nil),
214+
indexHeaderLoadDuration: prometheus.NewDesc(
215+
"cortex_bucket_store_indexheader_load_duration_seconds",
216+
"Duration of the index-header loading in seconds.",
217+
nil, nil),
208218

209219
lazyExpandedPostingsCount: prometheus.NewDesc(
210220
"cortex_bucket_store_lazy_expanded_postings_total",
@@ -272,6 +282,8 @@ func (m *BucketStoreMetrics) Describe(out chan<- *prometheus.Desc) {
272282
out <- m.indexHeaderLazyUnloadCount
273283
out <- m.indexHeaderLazyUnloadFailedCount
274284
out <- m.indexHeaderLazyLoadDuration
285+
out <- m.indexHeaderDownloadDuration
286+
out <- m.indexHeaderLoadDuration
275287

276288
out <- m.lazyExpandedPostingsCount
277289
out <- m.lazyExpandedPostingGroups
@@ -323,6 +335,8 @@ func (m *BucketStoreMetrics) Collect(out chan<- prometheus.Metric) {
323335
data.SendSumOfCounters(out, m.indexHeaderLazyUnloadCount, "thanos_bucket_store_indexheader_lazy_unload_total")
324336
data.SendSumOfCounters(out, m.indexHeaderLazyUnloadFailedCount, "thanos_bucket_store_indexheader_lazy_unload_failed_total")
325337
data.SendSumOfHistograms(out, m.indexHeaderLazyLoadDuration, "thanos_bucket_store_indexheader_lazy_load_duration_seconds")
338+
data.SendSumOfHistograms(out, m.indexHeaderDownloadDuration, "thanos_bucket_store_indexheader_download_duration_seconds")
339+
data.SendSumOfHistograms(out, m.indexHeaderLoadDuration, "thanos_bucket_store_indexheader_load_duration_seconds")
326340

327341
data.SendSumOfCounters(out, m.lazyExpandedPostingsCount, "thanos_bucket_store_lazy_expanded_postings_total")
328342
data.SendSumOfCountersWithLabels(out, m.lazyExpandedPostingGroups, "thanos_bucket_store_lazy_expanded_posting_groups_total", "reason")

‎pkg/storegateway/bucket_store_metrics_test.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -493,6 +493,22 @@ func TestBucketStoreMetrics(t *testing.T) {
493493
# HELP cortex_bucket_store_empty_postings_total Total number of empty postings when fetching block series.
494494
# TYPE cortex_bucket_store_empty_postings_total counter
495495
cortex_bucket_store_empty_postings_total 112595
496+
497+
# HELP cortex_bucket_store_indexheader_download_duration_seconds Duration of the index-header download from objstore in seconds.
498+
# TYPE cortex_bucket_store_indexheader_download_duration_seconds histogram
499+
cortex_bucket_store_indexheader_download_duration_seconds_bucket{le="0.01"} 0
500+
cortex_bucket_store_indexheader_download_duration_seconds_bucket{le="0.02"} 0
501+
cortex_bucket_store_indexheader_download_duration_seconds_bucket{le="0.05"} 0
502+
cortex_bucket_store_indexheader_download_duration_seconds_bucket{le="0.1"} 0
503+
cortex_bucket_store_indexheader_download_duration_seconds_bucket{le="0.2"} 0
504+
cortex_bucket_store_indexheader_download_duration_seconds_bucket{le="0.5"} 0
505+
cortex_bucket_store_indexheader_download_duration_seconds_bucket{le="1"} 3
506+
cortex_bucket_store_indexheader_download_duration_seconds_bucket{le="2"} 3
507+
cortex_bucket_store_indexheader_download_duration_seconds_bucket{le="5"} 3
508+
cortex_bucket_store_indexheader_download_duration_seconds_bucket{le="+Inf"} 3
509+
cortex_bucket_store_indexheader_download_duration_seconds_sum 2.25
510+
cortex_bucket_store_indexheader_download_duration_seconds_count 3
511+
496512
# HELP cortex_bucket_store_postings_fetch_duration_seconds Time it takes to fetch postings to respond a request sent to store-gateway. It includes both the time to fetch it from cache and from storage in case of cache misses.
497513
# TYPE cortex_bucket_store_postings_fetch_duration_seconds histogram
498514
cortex_bucket_store_postings_fetch_duration_seconds_bucket{le="0.001"} 0
@@ -543,6 +559,22 @@ func TestBucketStoreMetrics(t *testing.T) {
543559
# HELP cortex_bucket_store_indexheader_lazy_unload_total Total number of index-header lazy unload operations.
544560
# TYPE cortex_bucket_store_indexheader_lazy_unload_total counter
545561
cortex_bucket_store_indexheader_lazy_unload_total 1.396178e+06
562+
563+
# HELP cortex_bucket_store_indexheader_load_duration_seconds Duration of the index-header loading in seconds.
564+
# TYPE cortex_bucket_store_indexheader_load_duration_seconds histogram
565+
cortex_bucket_store_indexheader_load_duration_seconds_bucket{le="0.01"} 0
566+
cortex_bucket_store_indexheader_load_duration_seconds_bucket{le="0.02"} 0
567+
cortex_bucket_store_indexheader_load_duration_seconds_bucket{le="0.05"} 0
568+
cortex_bucket_store_indexheader_load_duration_seconds_bucket{le="0.1"} 0
569+
cortex_bucket_store_indexheader_load_duration_seconds_bucket{le="0.2"} 0
570+
cortex_bucket_store_indexheader_load_duration_seconds_bucket{le="0.5"} 0
571+
cortex_bucket_store_indexheader_load_duration_seconds_bucket{le="1"} 3
572+
cortex_bucket_store_indexheader_load_duration_seconds_bucket{le="2"} 3
573+
cortex_bucket_store_indexheader_load_duration_seconds_bucket{le="5"} 3
574+
cortex_bucket_store_indexheader_load_duration_seconds_bucket{le="+Inf"} 3
575+
cortex_bucket_store_indexheader_load_duration_seconds_sum 2.55
576+
cortex_bucket_store_indexheader_load_duration_seconds_count 3
577+
546578
# HELP cortex_bucket_store_lazy_expanded_posting_groups_total Total number of posting groups that are marked as lazy and corresponding reason.
547579
# TYPE cortex_bucket_store_lazy_expanded_posting_groups_total counter
548580
cortex_bucket_store_lazy_expanded_posting_groups_total{reason="keys_limit"} 202671
@@ -685,6 +717,8 @@ func populateMockedBucketStoreMetrics(base float64) *prometheus.Registry {
685717
m.indexHeaderLazyUnloadCount.Add(62 * base)
686718
m.indexHeaderLazyUnloadFailedCount.Add(63 * base)
687719
m.indexHeaderLazyLoadDuration.Observe(0.65)
720+
m.indexHeaderDownloadDuration.Observe(0.75)
721+
m.indexHeaderLoadDuration.Observe(0.85)
688722

689723
m.emptyPostingCount.Add(5 * base)
690724

@@ -737,6 +771,8 @@ type mockedBucketStoreMetrics struct {
737771
indexHeaderLazyUnloadCount prometheus.Counter
738772
indexHeaderLazyUnloadFailedCount prometheus.Counter
739773
indexHeaderLazyLoadDuration prometheus.Histogram
774+
indexHeaderDownloadDuration prometheus.Histogram
775+
indexHeaderLoadDuration prometheus.Histogram
740776

741777
lazyExpandedPostingsCount prometheus.Counter
742778
lazyExpandedPostingGroups *prometheus.CounterVec
@@ -913,6 +949,16 @@ func newMockedBucketStoreMetrics(reg prometheus.Registerer) *mockedBucketStoreMe
913949
Help: "Duration of the index-header lazy loading in seconds.",
914950
Buckets: []float64{0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5},
915951
})
952+
m.indexHeaderDownloadDuration = promauto.With(reg).NewHistogram(prometheus.HistogramOpts{
953+
Name: "thanos_bucket_store_indexheader_download_duration_seconds",
954+
Help: "Duration of the index-header download from objstore in seconds.",
955+
Buckets: []float64{0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5},
956+
})
957+
m.indexHeaderLoadDuration = promauto.With(reg).NewHistogram(prometheus.HistogramOpts{
958+
Name: "thanos_bucket_store_indexheader_load_duration_seconds",
959+
Help: "Duration of the index-header loading in seconds.",
960+
Buckets: []float64{0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5},
961+
})
916962

917963
m.emptyPostingCount = promauto.With(reg).NewCounter(prometheus.CounterOpts{
918964
Name: "thanos_bucket_store_empty_postings_total",

0 commit comments

Comments
 (0)