Add poll throttle latency metric

Author: madhuravi

common/metrics/defs.go

@@ -549,7 +549,8 @@ const (
 	LeaseFailureCounter
 	ConditionFailedErrorCounter
 	RespondQueryTaskFailedCounter
-	ThrottleErrorCounter
+	PollThrottleLatency
+	PollThrottleCounter
 )
 
 // MetricDefs record the metrics for all services
@@ -626,7 +627,8 @@ var MetricDefs = map[ServiceIdx]map[int]metricDefinition{
 		LeaseFailureCounter:           {metricName: "lease.failures"},
 		ConditionFailedErrorCounter:   {metricName: "condition-failed-errors"},
 		RespondQueryTaskFailedCounter: {metricName: "respond-query-failed"},
-		ThrottleErrorCounter:          {metricName: "throttle.errors"},
+		PollThrottleLatency:           {metricName: "poll.throttle.latency"},
+		PollThrottleCounter:           {metricName: "poll.throttle.count"},
 	},
 }
 

service/matching/matchingEngine_test.go

@@ -526,7 +526,7 @@ func (s *matchingEngineSuite) TestSyncMatchActivities() {
 		if maxDispatch == 0 {
 			wg.Wait()
 			s.Error(pollErr)
-			s.Contains(pollErr.Error(), "ServiceBusyError")
+			s.Contains(pollErr.Error(), "TaskList dispatch exceeded limit")
 			time.Sleep(dispatchTTL) // Sleep should be atleast ttl so max Dispatch gets updated
 			zeroDispatchCt++
 			continue
@@ -704,7 +704,7 @@ func (s *matchingEngineSuite) concurrentPublishConsumeActivities(
 					},
 				})
 				if err != nil {
-					s.Contains(err.Error(), "ServiceBusyError")
+					s.Contains(err.Error(), "TaskList dispatch exceeded limit")
 					throttleMu.Lock()
 					throttleCt++
 					throttleMu.Unlock()

service/matching/taskListManager.go

@@ -22,6 +22,7 @@ package matching
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"sync"
 	"sync/atomic"
@@ -409,11 +410,14 @@ func (c *taskListManagerImpl) getTask(ctx context.Context) (*getTaskResult, erro
 	}
 
 	// Wait till long poll expiration for token. If token acquired, proceed.
-	if err := c.rateLimiter.Wait(ctx, c.config.LongPollExpirationInterval); err != nil {
-		c.metricsClient.IncCounter(scope, metrics.ThrottleErrorCounter)
+	stopWatch := c.metricsClient.StartTimer(scope, metrics.PollThrottleLatency)
+	err := c.rateLimiter.Wait(ctx, c.config.LongPollExpirationInterval)
+	stopWatch.Stop()
+	if err != nil {
+		c.metricsClient.IncCounter(scope, metrics.PollThrottleCounter)
 		msg := fmt.Sprintf("TaskList dispatch exceeded limit: %s", err.Error())
 		// TODO: It's the client that's busy, not the server. Doesn't fit into any other error.
-		return nil, createServiceBusyError(msg)
+		return nil, errors.New(msg)
 	}
 	select {
 	case task, ok := <-c.taskBuffer: