From 47fe2c81195ba55e339b18608abaffedcdf663ab Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sun, 28 Jan 2024 14:54:21 -0800 Subject: [PATCH 1/3] add flag to enable store gateway query stats Signed-off-by: Ben Ye --- pkg/querier/blocks_store_queryable.go | 49 ++++++++++++++++----------- pkg/querier/querier.go | 6 ++-- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/pkg/querier/blocks_store_queryable.go b/pkg/querier/blocks_store_queryable.go index c4adf5bdab5..a8b0f0027cb 100644 --- a/pkg/querier/blocks_store_queryable.go +++ b/pkg/querier/blocks_store_queryable.go @@ -138,6 +138,8 @@ type BlocksStoreQueryable struct { metrics *blocksStoreQueryableMetrics limits BlocksStoreLimits + storeGatewayQueryStatsEnabled bool + // Subservices manager. subservices *services.Manager subservicesWatcher *services.FailureWatcher @@ -149,6 +151,7 @@ func NewBlocksStoreQueryable( consistency *BlocksConsistencyChecker, limits BlocksStoreLimits, queryStoreAfter time.Duration, + storeGatewayQueryStatsEnabled bool, logger log.Logger, reg prometheus.Registerer, ) (*BlocksStoreQueryable, error) { @@ -158,15 +161,16 @@ func NewBlocksStoreQueryable( } q := &BlocksStoreQueryable{ - stores: stores, - finder: finder, - consistency: consistency, - queryStoreAfter: queryStoreAfter, - logger: logger, - subservices: manager, - subservicesWatcher: services.NewFailureWatcher(), - metrics: newBlocksStoreQueryableMetrics(reg), - limits: limits, + stores: stores, + finder: finder, + consistency: consistency, + queryStoreAfter: queryStoreAfter, + logger: logger, + subservices: manager, + subservicesWatcher: services.NewFailureWatcher(), + metrics: newBlocksStoreQueryableMetrics(reg), + limits: limits, + storeGatewayQueryStatsEnabled: storeGatewayQueryStatsEnabled, } q.Service = services.NewBasicService(q.starting, q.running, q.stopping) @@ -256,7 +260,7 @@ func NewBlocksStoreQueryableFromConfig(querierCfg Config, gatewayCfg storegatewa reg, ) - return NewBlocksStoreQueryable(stores, finder, consistency, limits, querierCfg.QueryStoreAfter, logger, reg) + return NewBlocksStoreQueryable(stores, finder, consistency, limits, querierCfg.QueryStoreAfter, querierCfg.StoreGatewayQueryStatsEnabled, logger, reg) } func (q *BlocksStoreQueryable) starting(ctx context.Context) error { @@ -291,15 +295,16 @@ func (q *BlocksStoreQueryable) Querier(mint, maxt int64) (storage.Querier, error } return &blocksStoreQuerier{ - minT: mint, - maxT: maxt, - finder: q.finder, - stores: q.stores, - metrics: q.metrics, - limits: q.limits, - consistency: q.consistency, - logger: q.logger, - queryStoreAfter: q.queryStoreAfter, + minT: mint, + maxT: maxt, + finder: q.finder, + stores: q.stores, + metrics: q.metrics, + limits: q.limits, + consistency: q.consistency, + logger: q.logger, + queryStoreAfter: q.queryStoreAfter, + storeGatewayQueryStatsEnabled: q.storeGatewayQueryStatsEnabled, }, nil } @@ -315,6 +320,10 @@ type blocksStoreQuerier struct { // If set, the querier manipulates the max time to not be greater than // "now - queryStoreAfter" so that most recent blocks are not queried. queryStoreAfter time.Duration + + // If enabled, query stats of store gateway requests will be logged + // using `info` level. + storeGatewayQueryStatsEnabled bool } // Select implements storage.Querier interface. @@ -753,7 +762,7 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores( // Use number of blocks queried to check whether we should log the query // or not. It might be logging too much but good to understand per request // performance. - if seriesQueryStats.BlocksQueried > 0 { + if q.storeGatewayQueryStatsEnabled && seriesQueryStats.BlocksQueried > 0 { level.Info(spanLog).Log("msg", "store gateway series request stats", "instance", c.RemoteAddress(), "queryable_chunk_bytes_fetched", chunkBytes, diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 2a5e03af3b0..bcb0bd321fb 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -75,8 +75,9 @@ type Config struct { LookbackDelta time.Duration `yaml:"lookback_delta"` // Blocks storage only. - StoreGatewayAddresses string `yaml:"store_gateway_addresses"` - StoreGatewayClient ClientConfig `yaml:"store_gateway_client"` + StoreGatewayAddresses string `yaml:"store_gateway_addresses"` + StoreGatewayClient ClientConfig `yaml:"store_gateway_client"` + StoreGatewayQueryStatsEnabled bool `yaml:"store_gateway_query_stats"` ShuffleShardingIngestersLookbackPeriod time.Duration `yaml:"shuffle_sharding_ingesters_lookback_period"` @@ -112,6 +113,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.QueryStoreAfter, "querier.query-store-after", 0, "The time after which a metric should be queried from storage and not just ingesters. 0 means all queries are sent to store. When running the blocks storage, if this option is enabled, the time range of the query sent to the store will be manipulated to ensure the query end is not more recent than 'now - query-store-after'.") f.StringVar(&cfg.ActiveQueryTrackerDir, "querier.active-query-tracker-dir", "./active-query-tracker", "Active query tracker monitors active queries, and writes them to the file in given directory. If Cortex discovers any queries in this log during startup, it will log them to the log file. Setting to empty value disables active query tracker, which also disables -querier.max-concurrent option.") f.StringVar(&cfg.StoreGatewayAddresses, "querier.store-gateway-addresses", "", "Comma separated list of store-gateway addresses in DNS Service Discovery format. This option should be set when using the blocks storage and the store-gateway sharding is disabled (when enabled, the store-gateway instances form a ring and addresses are picked from the ring).") + f.BoolVar(&cfg.StoreGatewayQueryStatsEnabled, "querier.store-gateway-query-stats-enabled", true, "If enabled, store gateway query stats will be logged using `info` log level.") f.DurationVar(&cfg.LookbackDelta, "querier.lookback-delta", 5*time.Minute, "Time since the last sample after which a time series is considered stale and ignored by expression evaluations.") f.DurationVar(&cfg.ShuffleShardingIngestersLookbackPeriod, "querier.shuffle-sharding-ingesters-lookback-period", 0, "When distributor's sharding strategy is shuffle-sharding and this setting is > 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured 'query store after' and 'query ingesters within'. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled).") f.BoolVar(&cfg.ThanosEngine, "querier.thanos-engine", false, "Experimental. Use Thanos promql engine https://github.com/thanos-io/promql-engine rather than the Prometheus promql engine.") From 3a42ec1bcca0575a3c36bcbe8776a9725ede4c7c Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sun, 28 Jan 2024 23:07:42 +0000 Subject: [PATCH 2/3] docs update Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + docs/blocks-storage/querier.md | 4 ++++ docs/configuration/config-file-reference.md | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cd7776c557..6fb80db0d7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * [ENHANCEMENT] Index Cache: Multi level cache adds config `max_backfill_items` to cap max items to backfill per async operation. #5686 * [ENHANCEMENT] Query Frontend: Log number of split queries in `query stats` log. #5703 * [ENHANCEMENT] Logging: Added new options for logging HTTP request headers: `-server.log-request-headers` enables logging HTTP request headers, `-server.log-request-headers-exclude-list` allows users to specify headers which should not be logged. #5744 +* [ENHANCEMENT] Querier: Added `querier.store-gateway-query-stats-enabled` to enable or disable store gateway query stats log. #5749 * [BUGFIX] Distributor: Do not use label with empty values for sharding #5717 * [BUGFIX] Query Frontend: queries with negative offset should check whether it is cacheable or not. #5719 * [BUGFIX] Redis Cache: pass `cache_size` config correctly. #5734 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 4c303613ca7..aa7e138a501 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -214,6 +214,10 @@ querier: # CLI flag: -querier.store-gateway-client.grpc-compression [grpc_compression: | default = ""] + # If enabled, store gateway query stats will be logged using `info` log level. + # CLI flag: -querier.store-gateway-query-stats-enabled + [store_gateway_query_stats: | default = true] + # When distributor's sharding strategy is shuffle-sharding and this setting is # > 0, queriers fetch in-memory series from the minimum set of required # ingesters, selecting only ingesters which may have received series since diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index cac4fd15c22..efea8a6e1b5 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3527,6 +3527,10 @@ store_gateway_client: # CLI flag: -querier.store-gateway-client.grpc-compression [grpc_compression: | default = ""] +# If enabled, store gateway query stats will be logged using `info` log level. +# CLI flag: -querier.store-gateway-query-stats-enabled +[store_gateway_query_stats: | default = true] + # When distributor's sharding strategy is shuffle-sharding and this setting is > # 0, queriers fetch in-memory series from the minimum set of required ingesters, # selecting only ingesters which may have received series since 'now - lookback From 0bca57d71f22ed937a7d03050a94bba8da2953a6 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sun, 28 Jan 2024 20:47:10 -0800 Subject: [PATCH 3/3] fix tests Signed-off-by: Ben Ye --- pkg/querier/blocks_store_queryable_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/querier/blocks_store_queryable_test.go b/pkg/querier/blocks_store_queryable_test.go index 85de2f7615b..f624644866a 100644 --- a/pkg/querier/blocks_store_queryable_test.go +++ b/pkg/querier/blocks_store_queryable_test.go @@ -1588,7 +1588,7 @@ func TestBlocksStoreQuerier_PromQLExecution(t *testing.T) { } // Instance the querier that will be executed to run the query. - queryable, err := NewBlocksStoreQueryable(stores, finder, NewBlocksConsistencyChecker(0, 0, logger, nil), &blocksStoreLimitsMock{}, 0, logger, nil) + queryable, err := NewBlocksStoreQueryable(stores, finder, NewBlocksConsistencyChecker(0, 0, logger, nil), &blocksStoreLimitsMock{}, 0, false, logger, nil) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), queryable)) defer services.StopAndAwaitTerminated(context.Background(), queryable) // nolint:errcheck