Collect store gateway postings touched count and bytes in querier stats (#5892)

* update stats

Signed-off-by: Ben Ye <[email protected]>

* update stats

Signed-off-by: Ben Ye <[email protected]>

* collect store gateway postings touched count and bytes in querier stats

Signed-off-by: Ben Ye <[email protected]>

* changelog

Signed-off-by: Ben Ye <[email protected]>

---------

Signed-off-by: Ben Ye <[email protected]>

Author: yeya24

CHANGELOG.md

@@ -3,6 +3,8 @@
 ## master / unreleased
 * [CHANGE] Ruler: Remove `experimental.ruler.api-enable-rules-backup` flag and use `ruler.ring.replication-factor` to check if rules backup is enabled
 
+* [ENHANCEMENT] Query Frontend/Querier: Added store gateway postings touched count and touched size in Querier stats and log in Query Frontend. #5892
+
 ## 1.17.0 in progress
 
 * [CHANGE] Azure Storage: Upgraded objstore dependency and support Azure Workload Identity Authentication. Added `connection_string` to support authenticating via SAS token. Marked `msi_resource` config as deprecating. #5645

pkg/frontend/transport/handler.go

@@ -296,6 +296,8 @@ func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString u
 	numSamples := stats.LoadFetchedSamples()
 	numChunkBytes := stats.LoadFetchedChunkBytes()
 	numDataBytes := stats.LoadFetchedDataBytes()
+	numStoreGatewayTouchedPostings := stats.LoadStoreGatewayTouchedPostings()
+	numStoreGatewayTouchedPostingBytes := stats.LoadStoreGatewayTouchedPostingBytes()
 	splitQueries := stats.LoadSplitQueries()
 	dataSelectMaxTime := stats.LoadDataSelectMaxTime()
 	dataSelectMinTime := stats.LoadDataSelectMinTime()
@@ -334,6 +336,11 @@ func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString u
 		"response_size", contentLength,
 	}, stats.LoadExtraFields()...)
 
+	if numStoreGatewayTouchedPostings > 0 {
+		logMessage = append(logMessage, "store_gateway_touched_postings_count", numStoreGatewayTouchedPostings)
+		logMessage = append(logMessage, "store_gateway_touched_posting_bytes", numStoreGatewayTouchedPostingBytes)
+	}
+
 	grafanaFields := formatGrafanaStatsFields(r)
 	if len(grafanaFields) > 0 {
 		logMessage = append(logMessage, grafanaFields...)

pkg/frontend/transport/handler_test.go

@@ -471,6 +471,23 @@ func TestReportQueryStatsFormat(t *testing.T) {
 			},
 			expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 split_queries=0 status_code=200 response_size=1000 data_select_max_time=1704153600 data_select_min_time=1704067200 query_length=2 param_query=up`,
 		},
+		"should include query stats with store gateway stats": {
+			queryStats: &querier_stats.QueryStats{
+				Stats: querier_stats.Stats{
+					WallTime:                         3 * time.Second,
+					QueryStorageWallTime:             100 * time.Minute,
+					FetchedSeriesCount:               100,
+					FetchedChunksCount:               200,
+					FetchedSamplesCount:              300,
+					FetchedChunkBytes:                1024,
+					FetchedDataBytes:                 2048,
+					SplitQueries:                     10,
+					StoreGatewayTouchedPostingsCount: 20,
+					StoreGatewayTouchedPostingBytes:  200,
+				},
+			},
+			expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=3 fetched_series_count=100 fetched_chunks_count=200 fetched_samples_count=300 fetched_chunks_bytes=1024 fetched_data_bytes=2048 split_queries=10 status_code=200 response_size=1000 store_gateway_touched_postings_count=20 store_gateway_touched_posting_bytes=200 query_storage_wall_time_seconds=6000`,
+		},
 	}
 
 	for testName, testData := range tests {

pkg/querier/blocks_store_queryable.go

@@ -756,6 +756,8 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores(
 			reqStats.AddFetchedSamples(numSamples)
 			reqStats.AddFetchedChunkBytes(uint64(chunkBytes))
 			reqStats.AddFetchedDataBytes(uint64(dataBytes))
+			reqStats.AddStoreGatewayTouchedPostings(uint64(seriesQueryStats.PostingsTouched))
+			reqStats.AddStoreGatewayTouchedPostingBytes(uint64(seriesQueryStats.PostingsTouchedSizeSum))
 
 			level.Debug(spanLog).Log("msg", "received series from store-gateway",
 				"instance", c.RemoteAddress(),

pkg/querier/stats/stats.go

@@ -270,6 +270,38 @@ func (s *QueryStats) LoadDataSelectMinTime() int64 {
 	return atomic.LoadInt64(&s.DataSelectMinTime)
 }
 
+func (s *QueryStats) AddStoreGatewayTouchedPostings(count uint64) {
+	if s == nil {
+		return
+	}
+
+	atomic.AddUint64(&s.StoreGatewayTouchedPostingsCount, count)
+}
+
+func (s *QueryStats) LoadStoreGatewayTouchedPostings() uint64 {
+	if s == nil {
+		return 0
+	}
+
+	return atomic.LoadUint64(&s.StoreGatewayTouchedPostingsCount)
+}
+
+func (s *QueryStats) AddStoreGatewayTouchedPostingBytes(bytes uint64) {
+	if s == nil {
+		return
+	}
+
+	atomic.AddUint64(&s.StoreGatewayTouchedPostingBytes, bytes)
+}
+
+func (s *QueryStats) LoadStoreGatewayTouchedPostingBytes() uint64 {
+	if s == nil {
+		return 0
+	}
+
+	return atomic.LoadUint64(&s.StoreGatewayTouchedPostingBytes)
+}
+
 // Merge the provided Stats into this one.
 func (s *QueryStats) Merge(other *QueryStats) {
 	if s == nil || other == nil {
@@ -283,6 +315,8 @@ func (s *QueryStats) Merge(other *QueryStats) {
 	s.AddFetchedDataBytes(other.LoadFetchedDataBytes())
 	s.AddFetchedSamples(other.LoadFetchedSamples())
 	s.AddFetchedChunks(other.LoadFetchedChunks())
+	s.AddStoreGatewayTouchedPostings(other.LoadStoreGatewayTouchedPostings())
+	s.AddStoreGatewayTouchedPostingBytes(other.LoadStoreGatewayTouchedPostingBytes())
 	s.AddExtraFields(other.LoadExtraFields()...)
 }
 

pkg/querier/stats/stats.pb.go

@@ -33,4 +33,10 @@ message Stats {
   uint64 split_queries = 9;
   // The sum of wall time spent in the querier to fetch and merge data from storage.
   google.protobuf.Duration query_storage_wall_time = 10 [(gogoproto.stdduration) = true, (gogoproto.nullable) = false];
+  // The total number of postings touched in store gateway for a specific query.
+  // Only successful requests from querier to store gateway are included.
+  uint64 store_gateway_touched_postings_count = 11;
+  // The total size of postings touched in store gateway for a specific query, in bytes.
+  // Only successful requests from querier to store gateway are included.
+  uint64 store_gateway_touched_posting_bytes = 12;
 }