Re-enqueue 429 requests if there are multiple query-schedulers

Signed-off-by: Xiaochao Dong (@damnever) <[email protected]>

Author: damnever

pkg/frontend/v2/frontend.go

@@ -86,6 +86,8 @@ type frontendRequest struct {
 
 	enqueue  chan enqueueResult
 	response chan *frontendv2pb.QueryResultRequest
+
+	retryOnTooManyOutstandingRequests bool
 }
 
 type enqueueStatus int
@@ -192,6 +194,8 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest)
 		// even if this goroutine goes away due to client context cancellation.
 		enqueue:  make(chan enqueueResult, 1),
 		response: make(chan *frontendv2pb.QueryResultRequest, 1),
+
+		retryOnTooManyOutstandingRequests: f.schedulerWorkers.getWorkersCount() > 0,
 	}
 
 	f.requests.put(freq)

pkg/frontend/v2/frontend_scheduler_worker.go

@@ -296,12 +296,16 @@ func (w *frontendSchedulerWorker) schedulerLoop(loop schedulerpb.SchedulerForFro
 				}
 
 			case schedulerpb.TOO_MANY_REQUESTS_PER_TENANT:
-				req.enqueue <- enqueueResult{status: waitForResponse}
-				req.response <- &frontendv2pb.QueryResultRequest{
-					HttpResponse: &httpgrpc.HTTPResponse{
-						Code: http.StatusTooManyRequests,
-						Body: []byte("too many outstanding requests"),
-					},
+				if req.retryOnTooManyOutstandingRequests {
+					req.enqueue <- enqueueResult{status: failed}
+				} else {
+					req.enqueue <- enqueueResult{status: waitForResponse}
+					req.response <- &frontendv2pb.QueryResultRequest{
+						HttpResponse: &httpgrpc.HTTPResponse{
+							Code: http.StatusTooManyRequests,
+							Body: []byte("too many outstanding requests"),
+						},
+					}
 				}
 			}