Add counter metrics for total requests going to queue (#5030)

* add counter metrics for total requests going to queue

Signed-off-by: Ben Ye <[email protected]>

* update changelog

Signed-off-by: Ben Ye <[email protected]>

* fix lint

Signed-off-by: Ben Ye <[email protected]>

* lint

Signed-off-by: Ben Ye <[email protected]>

Signed-off-by: Ben Ye <[email protected]>

Author: yeya24

CHANGELOG.md

@@ -11,6 +11,7 @@
 * [FEATURE] Ingester: Added `-blocks-storage.tsdb.head-chunks-write-queue-size` allowing to configure the size of the in-memory queue used before flushing chunks to the disk . #5000
 * [FEATURE] Query Frontend: Log query params in query frontend even if error happens. #5005
 * [FEATURE] Ingester: Enable snapshotting of In-memory TSDB on disk during shutdown via `-blocks-storage.tsdb.memory-snapshot-on-shutdown`. #5011
+* [FEATURE] Query Frontend/Scheduler: Add a new counter metric `cortex_request_queue_requests_total` for total requests going to queue. #5030
 * [BUGFIX] Updated `golang.org/x/net` dependency to fix CVE-2022-27664. #5008
 
 ## 1.14.0 2022-12-02

pkg/frontend/v1/frontend.go

@@ -112,7 +112,7 @@ func New(cfg Config, limits Limits, log log.Logger, registerer prometheus.Regist
 		}),
 	}
 
-	f.requestQueue = queue.NewRequestQueue(cfg.MaxOutstandingPerTenant, cfg.QuerierForgetDelay, f.queueLength, f.discardedRequests, f.limits)
+	f.requestQueue = queue.NewRequestQueue(cfg.MaxOutstandingPerTenant, cfg.QuerierForgetDelay, f.queueLength, f.discardedRequests, f.limits, registerer)
 	f.activeUsers = util.NewActiveUsersCleanupWithDefaultValues(f.cleanupInactiveUserMetrics)
 
 	var err error

pkg/frontend/v1/frontend_test.go

@@ -133,6 +133,7 @@ func TestFrontendCheckReady(t *testing.T) {
 					prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
 					prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
 					limits,
+					nil,
 				),
 			}
 			for i := 0; i < tt.connectedClients; i++ {

pkg/scheduler/queue/queue.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"go.uber.org/atomic"
 
 	"github.com/cortexproject/cortex/pkg/util/services"
@@ -58,15 +59,20 @@ type RequestQueue struct {
 	stopped bool
 
 	queueLength       *prometheus.GaugeVec   // Per user and reason.
+	totalRequests     *prometheus.CounterVec // Per user.
 	discardedRequests *prometheus.CounterVec // Per user.
 }
 
-func NewRequestQueue(maxOutstandingPerTenant int, forgetDelay time.Duration, queueLength *prometheus.GaugeVec, discardedRequests *prometheus.CounterVec, limits Limits) *RequestQueue {
+func NewRequestQueue(maxOutstandingPerTenant int, forgetDelay time.Duration, queueLength *prometheus.GaugeVec, discardedRequests *prometheus.CounterVec, limits Limits, registerer prometheus.Registerer) *RequestQueue {
 	q := &RequestQueue{
 		queues:                  newUserQueues(maxOutstandingPerTenant, forgetDelay, limits),
 		connectedQuerierWorkers: atomic.NewInt32(0),
 		queueLength:             queueLength,
-		discardedRequests:       discardedRequests,
+		totalRequests: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
+			Name: "cortex_request_queue_requests_total",
+			Help: "Total number of query requests going to the request queue.",
+		}, []string{"user"}),
+		discardedRequests: discardedRequests,
 	}
 
 	q.cond = sync.NewCond(&q.mtx)
@@ -94,6 +100,7 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers in
 		return errors.New("no queue found")
 	}
 
+	q.totalRequests.WithLabelValues(userID).Inc()
 	select {
 	case queue <- req:
 		q.queueLength.WithLabelValues(userID).Inc()

pkg/scheduler/queue/queue_test.go

@@ -27,6 +27,7 @@ func BenchmarkGetNextRequest(b *testing.B) {
 			prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
 			prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
 			MockLimits{MaxOutstanding: 100},
+			nil,
 		)
 		queues = append(queues, queue)
 
@@ -85,6 +86,7 @@ func BenchmarkQueueRequest(b *testing.B) {
 			prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
 			prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
 			MockLimits{MaxOutstanding: 100},
+			nil,
 		)
 
 		for ix := 0; ix < queriers; ix++ {
@@ -119,6 +121,7 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe
 		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
 		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
 		MockLimits{MaxOutstanding: 100},
+		nil,
 	)
 
 	// Start the queue service.

pkg/scheduler/scheduler.go

@@ -111,7 +111,8 @@ func NewScheduler(cfg Config, limits Limits, log log.Logger, registerer promethe
 		Name: "cortex_query_scheduler_discarded_requests_total",
 		Help: "Total number of query requests discarded.",
 	}, []string{"user"})
-	s.requestQueue = queue.NewRequestQueue(cfg.MaxOutstandingPerTenant, cfg.QuerierForgetDelay, s.queueLength, s.discardedRequests, s.limits)
+
+	s.requestQueue = queue.NewRequestQueue(cfg.MaxOutstandingPerTenant, cfg.QuerierForgetDelay, s.queueLength, s.discardedRequests, s.limits, registerer)
 
 	s.queueDuration = promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{
 		Name:    "cortex_query_scheduler_queue_duration_seconds",