Removing a specific config for list rules fanout timeout and introducing generic remote_timeout

Signed-off-by: Anand Rajagopal <[email protected]>

Author: rajagopalanand

docs/configuration/config-file-reference.md

@@ -4142,6 +4142,10 @@ ruler_client:
   # CLI flag: -ruler.client.tls-insecure-skip-verify
   [tls_insecure_skip_verify: <boolean> | default = false]
 
+  # Timeout for downstream rulers.
+  # CLI flag: -ruler.client.remote-timeout
+  [remote_timeout: <duration> | default = 2m]
+
 # How frequently to evaluate rules
 # CLI flag: -ruler.evaluation-interval
 [evaluation_interval: <duration> | default = 1m]
@@ -4382,10 +4386,6 @@ ring:
 # Enable high availability
 # CLI flag: -ruler.enable-ha-evaluation
 [enable_ha_evaluation: <boolean> | default = false]
-
-# Timeout for fanout calls to other rulers
-# CLI flag: -ruler.list-rules-fanout-timeout
-[list_rules_fanout_timeout: <duration> | default = 2m]
 ```
 
 ### `ruler_storage_config`

integration/ruler_test.go

@@ -1040,7 +1040,7 @@ func TestRulerHA(t *testing.T) {
 		"-ruler.ring.replication-factor":                    "2",
 		"-ruler.enable-ha-evaluation":                       "true",
 		"-ruler.poll-interval":                              "5s",
-		"-ruler.list-rules-fanout-timeout":                  "10ms",
+		"-ruler.client.remote-timeout":                      "10ms",
 	}
 
 	rulerFlags := mergeFlags(

pkg/ruler/ruler.go

@@ -37,7 +37,6 @@ import (
 	util_api "github.com/cortexproject/cortex/pkg/util/api"
 	"github.com/cortexproject/cortex/pkg/util/concurrency"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
-	"github.com/cortexproject/cortex/pkg/util/grpcclient"
 	util_log "github.com/cortexproject/cortex/pkg/util/log"
 	"github.com/cortexproject/cortex/pkg/util/services"
 	"github.com/cortexproject/cortex/pkg/util/validation"
@@ -100,7 +99,7 @@ type Config struct {
 	// Labels to add to all alerts
 	ExternalLabels labels.Labels `yaml:"external_labels,omitempty" doc:"nocli|description=Labels to add to all alerts."`
 	// GRPC Client configuration.
-	ClientTLSConfig grpcclient.Config `yaml:"ruler_client"`
+	ClientTLSConfig ClientConfig `yaml:"ruler_client"`
 	// How frequently to evaluate rules by default.
 	EvaluationInterval time.Duration `yaml:"evaluation_interval"`
 	// How frequently to poll for updated rules.
@@ -154,8 +153,7 @@ type Config struct {
 	EnableQueryStats      bool `yaml:"query_stats_enabled"`
 	DisableRuleGroupLabel bool `yaml:"disable_rule_group_label"`
 
-	EnableHAEvaluation     bool          `yaml:"enable_ha_evaluation"`
-	ListRulesFanoutTimeout time.Duration `yaml:"list_rules_fanout_timeout"`
+	EnableHAEvaluation bool `yaml:"enable_ha_evaluation"`
 }
 
 // Validate config and returns error on failure
@@ -180,7 +178,7 @@ func (cfg *Config) Validate(limits validation.Limits, log log.Logger) error {
 
 // RegisterFlags adds the flags required to config this to the given FlagSet
 func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
-	cfg.ClientTLSConfig.RegisterFlagsWithPrefix("ruler.client", "", f)
+	cfg.ClientTLSConfig.RegisterFlagsWithPrefix("ruler.client", f)
 	cfg.Ring.RegisterFlags(f)
 	cfg.Notifier.RegisterFlags(f)
 
@@ -226,7 +224,6 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.DisableRuleGroupLabel, "ruler.disable-rule-group-label", false, "Disable the rule_group label on exported metrics")
 
 	f.BoolVar(&cfg.EnableHAEvaluation, "ruler.enable-ha-evaluation", false, "Enable high availability")
-	f.DurationVar(&cfg.ListRulesFanoutTimeout, "ruler.list-rules-fanout-timeout", 2*time.Minute, "Timeout for fanout calls to other rulers")
 
 	cfg.RingCheckPeriod = 5 * time.Second
 }
@@ -312,7 +309,7 @@ type Ruler struct {
 
 // NewRuler creates a new ruler from a distributor and chunk store.
 func NewRuler(cfg Config, manager MultiTenantManager, reg prometheus.Registerer, logger log.Logger, ruleStore rulestore.RuleStore, limits RulesLimits) (*Ruler, error) {
-	return newRuler(cfg, manager, reg, logger, ruleStore, limits, newRulerClientPool(cfg.ClientTLSConfig, logger, reg))
+	return newRuler(cfg, manager, reg, logger, ruleStore, limits, newRulerClientPool(cfg.ClientTLSConfig.Config, logger, reg))
 }
 
 func newRuler(cfg Config, manager MultiTenantManager, reg prometheus.Registerer, logger log.Logger, ruleStore rulestore.RuleStore, limits RulesLimits, clientPool ClientsPool) (*Ruler, error) {
@@ -1300,7 +1297,7 @@ func (r *Ruler) getShardedRules(ctx context.Context, userID string, rulesRequest
 			return errors.Wrapf(err, "unable to get client for ruler %s", addr)
 		}
 
-		ctx, cancel := context.WithTimeout(ctx, r.cfg.ListRulesFanoutTimeout)
+		ctx, cancel := context.WithTimeout(ctx, r.cfg.ClientTLSConfig.RemoteTimeout)
 		defer cancel()
 		newGrps, err := rulerClient.Rules(ctx, &RulesRequest{
 			RuleNames:      rulesRequest.GetRuleNames(),

pkg/ruler/ruler_client_config.go

@@ -0,0 +1,18 @@
+package ruler
+
+import (
+	"flag"
+	"time"
+
+	"github.com/cortexproject/cortex/pkg/util/grpcclient"
+)
+
+type ClientConfig struct {
+	grpcclient.Config `yaml:",inline"`
+	RemoteTimeout     time.Duration `yaml:"remote_timeout"`
+}
+
+func (cfg *ClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	cfg.Config.RegisterFlagsWithPrefix(prefix, "", f)
+	f.DurationVar(&cfg.RemoteTimeout, prefix+".remote-timeout", 2*time.Minute, "Timeout for downstream rulers.")
+}

pkg/ruler/ruler_test.go

@@ -229,7 +229,7 @@ func (p *mockRulerClientsPool) GetClientFor(addr string) (RulerClient, error) {
 
 func newMockClientsPool(cfg Config, logger log.Logger, reg prometheus.Registerer, rulerAddrMap map[string]*Ruler) *mockRulerClientsPool {
 	return &mockRulerClientsPool{
-		ClientsPool:  newRulerClientPool(cfg.ClientTLSConfig, logger, reg),
+		ClientsPool:  newRulerClientPool(cfg.ClientTLSConfig.Config, logger, reg),
 		cfg:          cfg,
 		rulerAddrMap: rulerAddrMap,
 	}