Little query tee enhancements (#2799)

* Higher granularity for the query-tee duration metrics

Signed-off-by: Marco Pracucci <[email protected]>

* Fast fail if the preferred backend is misconfigured

Signed-off-by: Marco Pracucci <[email protected]>

* Added CHANGELOG entry

Signed-off-by: Marco Pracucci <[email protected]>

Author: pracucci

CHANGELOG.md

@@ -11,6 +11,8 @@
 * [FEATURE] Introduced `ruler.for-grace-period`, Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. #2783
 * [FEATURE] Introduced `ruler.resend-delay`, Minimum amount of time to wait before resending an alert to Alertmanager. #2783
 * [ENHANCEMENT] Experimental: Querier can now optionally query secondary store. This is specified by using `-querier.second-store-engine` option, with values `chunks` or `tsdb`. Standard configuration options for this store are used. Additionally, this querying can be configured to happen only for queries that need data older than `-querier.use-second-store-before-time`. Default value of zero will always query secondary store. #2747
+* [ENHANCEMENT] Query-tee: increased the `cortex_querytee_request_duration_seconds` metric buckets granularity. #2799
+* [ENHANCEMENT] Query-tee: fail to start if the configured `-backend.preferred` is unknown. #2799
 * [ENHANCEMENT] Ruler: Added the following metrics: #2786
   * `cortex_prometheus_notifications_latency_seconds`
   * `cortex_prometheus_notifications_errors_total`

tools/querytee/proxy.go

@@ -107,6 +107,21 @@ func NewProxy(cfg ProxyConfig, logger log.Logger, routes []Route, registerer pro
 		return nil, errMinBackends
 	}
 
+	// If the preferred backend is configured, then it must exists among the actual backends.
+	if cfg.PreferredBackend != "" {
+		exists := false
+		for _, b := range p.backends {
+			if b.preferred {
+				exists = true
+				break
+			}
+		}
+
+		if !exists {
+			return nil, fmt.Errorf("the preferred backend (hostname) has not been found among the list of configured backends")
+		}
+	}
+
 	if cfg.CompareResponses && len(p.backends) != 2 {
 		return nil, fmt.Errorf("when enabling comparison of results number of backends should be 2 exactly")
 	}
@@ -159,6 +174,7 @@ func (p *Proxy) Start() error {
 		}
 	}()
 
+	level.Info(p.logger).Log("msg", "The proxy is up and running.")
 	return nil
 }
 

tools/querytee/proxy_metrics.go

@@ -3,7 +3,6 @@ package querytee
 import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/weaveworks/common/instrument"
 )
 
 const (
@@ -23,7 +22,7 @@ func NewProxyMetrics(registerer prometheus.Registerer) *ProxyMetrics {
 			Namespace: "cortex_querytee",
 			Name:      "request_duration_seconds",
 			Help:      "Time (in seconds) spent serving HTTP requests.",
-			Buckets:   instrument.DefBuckets,
+			Buckets:   []float64{.005, .01, .025, .05, .1, .25, .5, 0.75, 1, 1.5, 2, 3, 4, 5, 10, 25, 50, 100},
 		}, []string{"backend", "method", "route", "status_code"}),
 		responsesTotal: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
 			Namespace: "cortex_querytee",