Add limit breached metric + wrap error with 429

Signed-off-by: Justin Jung <[email protected]>

Author: justinjung04

pkg/ingester/ingester.go

@@ -2172,7 +2172,7 @@ func (i *Ingester) trackInflightQueryRequest() (func(), error) {
 	if i.resourceBasedLimiter != nil {
 		if err := i.resourceBasedLimiter.AcceptNewRequest(); err != nil {
 			level.Warn(i.logger).Log("msg", "failed to accept request", "err", err)
-			return nil, fmt.Errorf("failed to query: %s", limiter.ErrResourceLimitReachedStr)
+			return nil, httpgrpc.Errorf(http.StatusTooManyRequests, "failed to query: %s", limiter.ErrResourceLimitReachedStr)
 		}
 	}
 

pkg/storegateway/gateway.go

@@ -16,6 +16,7 @@ import (
 	"github.com/thanos-io/objstore"
 	"github.com/thanos-io/thanos/pkg/extprom"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
+	"github.com/weaveworks/common/httpgrpc"
 	"github.com/weaveworks/common/logging"
 
 	"github.com/cortexproject/cortex/pkg/configs"
@@ -436,7 +437,7 @@ func (g *StoreGateway) checkResourceUtilization() error {
 
 	if err := g.resourceBasedLimiter.AcceptNewRequest(); err != nil {
 		level.Warn(g.logger).Log("msg", "failed to accept request", "err", err)
-		return fmt.Errorf("failed to query: %s", util_limiter.ErrResourceLimitReachedStr)
+		return httpgrpc.Errorf(http.StatusTooManyRequests, "failed to query: %s", util_limiter.ErrResourceLimitReachedStr)
 	}
 
 	return nil

pkg/util/limiter/resource_based_limiter.go

@@ -18,8 +18,9 @@ func (e *ResourceLimitReachedError) Error() string {
 }
 
 type ResourceBasedLimiter struct {
-	resourceMonitor resource.IMonitor
-	limits          map[resource.Type]float64
+	resourceMonitor    resource.IMonitor
+	limits             map[resource.Type]float64
+	limitBreachedCount *prometheus.CounterVec
 }
 
 func NewResourceBasedLimiter(resourceMonitor resource.IMonitor, limits map[resource.Type]float64, registerer prometheus.Registerer) (*ResourceBasedLimiter, error) {
@@ -38,6 +39,13 @@ func NewResourceBasedLimiter(resourceMonitor resource.IMonitor, limits map[resou
 	return &ResourceBasedLimiter{
 		resourceMonitor: resourceMonitor,
 		limits:          limits,
+		limitBreachedCount: promauto.With(registerer).NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "cortex_resource_based_limiter_limit_breached",
+				Help: "The total number of times resource based limiter was throttled.",
+			},
+			[]string{"resource"},
+		),
 	}, nil
 }
 
@@ -53,6 +61,7 @@ func (l *ResourceBasedLimiter) AcceptNewRequest() error {
 		}
 
 		if utilization >= limit {
+			l.limitBreachedCount.WithLabelValues(string(resType)).Inc()
 			return fmt.Errorf("%s utilization limit reached (limit: %.3f, utilization: %.3f)", resType, limit, utilization)
 		}
 	}