Add stats log when querying store gateway (#5376)

* add stats log when querying store gateway

Signed-off-by: Ben Ye <[email protected]>

* update changelog

Signed-off-by: Ben Ye <[email protected]>

* add data_downloaded_size_sum

Signed-off-by: Ben Ye <[email protected]>

* update vendor

Signed-off-by: Ben Ye <[email protected]>

---------

Signed-off-by: Ben Ye <[email protected]>

Author: yeya24

CHANGELOG.md

@@ -12,6 +12,7 @@
 * [FEATURE] Store Gateway: Add `max_downloaded_bytes_per_request` to limit max bytes to download per store gateway request.
 * [FEATURE] Added 2 flags `-alertmanager.alertmanager-client.grpc-max-send-msg-size` and ` -alertmanager.alertmanager-client.grpc-max-recv-msg-size` to configure alert manager grpc client message size limits. #5338
 * [FEATURE] Query Frontend: Add `cortex_rejected_queries_total` metric for throttled queries. #5356
+* [FEATURE] Querier: Log query stats when querying store gateway. #5376
 * [ENHANCEMENT] Distributor/Ingester: Add span on push path #5319
 * [ENHANCEMENT] Support object storage backends for runtime configuration file. #5292
 * [ENHANCEMENT] Query Frontend: Reject subquery with too small step size. #5323

go.mod

@@ -52,8 +52,8 @@ require (
 	github.com/spf13/afero v1.9.5
 	github.com/stretchr/testify v1.8.3
 	github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204
-	github.com/thanos-io/promql-engine v0.0.0-20230524173220-ac7dd841e96a
-	github.com/thanos-io/thanos v0.0.0-20230524201324-c4321c5e1de2
+	github.com/thanos-io/promql-engine v0.0.0-20230526105742-791d78b260ea
+	github.com/thanos-io/thanos v0.31.1-0.20230602190045-fbc88857d78c
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d
 	go.etcd.io/etcd/api/v3 v3.5.8

go.sum

@@ -849,7 +849,7 @@ github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
 github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
 github.com/efficientgo/core v1.0.0-rc.2 h1:7j62qHLnrZqO3V3UA0AqOGd5d5aXV3AX6m/NZBHp78I=
 github.com/efficientgo/core v1.0.0-rc.2/go.mod h1:FfGdkzWarkuzOlY04VY+bGfb1lWrjaL6x/GLcQ4vJps=
-github.com/efficientgo/e2e v0.14.1-0.20230413163036-7a7e0bae9913 h1:/CcqWjW1rMnoYiiKpiVKcReOJCGnCo868z85g7TwRhg=
+github.com/efficientgo/e2e v0.14.1-0.20230530075226-84bb33e062c4 h1:8w29+abukpj2UWN19wZ4xmOswdrYBjfqmbot9oppp3U=
 github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd h1:VaYzzXeUbC5fVheskcKVNOyJMEYD+HgrJNzIAg/mRIM=
 github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd/go.mod h1:ZV0utlglOczUWv3ih2AbqPSoLoFzdplUYxwV62eZi6Q=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
@@ -1867,10 +1867,10 @@ github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1
 github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM=
 github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204 h1:W4w5Iph7j32Sf1QFWLJDCqvO0WgZS0jHGID+qnq3wV0=
 github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204/go.mod h1:STSgpY8M6EKF2G/raUFdbIMf2U9GgYlEjAEHJxjvpAo=
-github.com/thanos-io/promql-engine v0.0.0-20230524173220-ac7dd841e96a h1:B0QFv7nykRtUt6YfYtHs8k96bZt8+HnqbZRzPutil80=
-github.com/thanos-io/promql-engine v0.0.0-20230524173220-ac7dd841e96a/go.mod h1:eIgPaXWgOhNAv6CPPrgu09r0AtT7byBTZy+7WkX0D18=
-github.com/thanos-io/thanos v0.0.0-20230524201324-c4321c5e1de2 h1:8unrRqovq5JbPqAuYoR9gxRZnxlkE5XesXBiJ5sg800=
-github.com/thanos-io/thanos v0.0.0-20230524201324-c4321c5e1de2/go.mod h1:VgKd9z92eKZnZDjqhLwEkYryWzVwNBJgUES9gA6Ky/E=
+github.com/thanos-io/promql-engine v0.0.0-20230526105742-791d78b260ea h1:kzK8sBn2+mo3NAxP+XjAjAqr1hwfxxFUy5CybaBkjAI=
+github.com/thanos-io/promql-engine v0.0.0-20230526105742-791d78b260ea/go.mod h1:eIgPaXWgOhNAv6CPPrgu09r0AtT7byBTZy+7WkX0D18=
+github.com/thanos-io/thanos v0.31.1-0.20230602190045-fbc88857d78c h1:XlhPcKRo4eQ5He/2+8Vfmfkxz+1skwEBuYVgMFBqePE=
+github.com/thanos-io/thanos v0.31.1-0.20230602190045-fbc88857d78c/go.mod h1:lHSiSsXIQuAv5u+6yu0LLw6cS/MC8vUQswQ6rkdxB7c=
 github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU=
 github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

pkg/querier/blocks_store_queryable.go

@@ -583,7 +583,6 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores(
 
 	if err != nil {
 		return nil, nil, nil, 0, err
-
 	}
 	convertedMatchers := convertMatchersToLabelMatcher(matchers)
 
@@ -601,6 +600,7 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores(
 
 			// Only fail the function if we have validation error. We should return blocks that were successfully
 			// retrieved.
+			seriesQueryStats := &hintspb.QueryStats{}
 			skipChunks := sp != nil && sp.Func == "series"
 
 			req, err := createSeriesRequest(minT, maxT, convertedMatchers, shardingInfo, skipChunks, blockIDs)
@@ -698,6 +698,9 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores(
 					}
 
 					myQueriedBlocks = append(myQueriedBlocks, ids...)
+					if hints.QueryStats != nil {
+						seriesQueryStats.Merge(hints.QueryStats)
+					}
 				}
 			}
 
@@ -714,14 +717,35 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores(
 
 			level.Debug(spanLog).Log("msg", "received series from store-gateway",
 				"instance", c.RemoteAddress(),
-				"fetched series", numSeries,
-				"fetched chunks", chunksCount,
-				"fetched samples", numSamples,
-				"fetched chunk bytes", chunkBytes,
-				"fetched data bytes", dataBytes,
 				"requested blocks", strings.Join(convertULIDsToString(blockIDs), " "),
 				"queried blocks", strings.Join(convertULIDsToString(myQueriedBlocks), " "))
 
+			level.Info(spanLog).Log("msg", "store gateway series request stats",
+				"instance", c.RemoteAddress(),
+				"queryable_chunk_bytes_fetched", chunkBytes,
+				"queryable_data_bytes_fetched", dataBytes,
+				"blocks_queried", seriesQueryStats.BlocksQueried,
+				"series_merged_count", seriesQueryStats.MergedSeriesCount,
+				"chunks_merged_count", seriesQueryStats.MergedChunksCount,
+				"postings_touched", seriesQueryStats.PostingsTouched,
+				"postings_touched_size_sum", seriesQueryStats.PostingsTouchedSizeSum,
+				"postings_to_fetch", seriesQueryStats.PostingsToFetch,
+				"postings_fetched", seriesQueryStats.PostingsFetched,
+				"postings_fetch_count", seriesQueryStats.PostingsFetchCount,
+				"postings_fetched_size_sum", seriesQueryStats.PostingsFetchedSizeSum,
+				"series_touched", seriesQueryStats.SeriesTouched,
+				"series_touched_size_sum", seriesQueryStats.SeriesTouchedSizeSum,
+				"series_fetched", seriesQueryStats.SeriesFetched,
+				"series_fetch_count", seriesQueryStats.SeriesFetchCount,
+				"series_fetched_size_sum", seriesQueryStats.SeriesFetchedSizeSum,
+				"chunks_touched", seriesQueryStats.ChunksTouched,
+				"chunks_touched_size_sum", seriesQueryStats.ChunksTouchedSizeSum,
+				"chunks_fetched", seriesQueryStats.ChunksFetched,
+				"chunks_fetch_count", seriesQueryStats.ChunksFetchCount,
+				"chunks_fetched_size_sum", seriesQueryStats.ChunksFetchedSizeSum,
+				"data_downloaded_size_sum", seriesQueryStats.DataDownloadedSizeSum,
+			)
+
 			// Store the result.
 			mtx.Lock()
 			seriesSets = append(seriesSets, &blockQuerierSeriesSet{series: mySeries})
@@ -937,6 +961,7 @@ func createSeriesRequest(minT, maxT int64, matchers []storepb.LabelMatcher, shar
 				Value: strings.Join(convertULIDsToString(blockIDs), "|"),
 			},
 		},
+		EnableQueryStats: true,
 	}
 
 	anyHints, err := types.MarshalAny(hints)