Ability to customize PVC workqueue retry interval

README.md

@@ -47,6 +47,10 @@ Note that the external-resizer does not scale with more replicas. Only one exter
 
 * `--csiTimeout <duration>`: Timeout of all calls to CSI driver. It should be set to value that accommodates majority of `ControllerExpandVolume` calls. 15 seconds is used by default.
 
+* `--retry-interval-start`: The starting value of the exponential backoff for failures. 1 second is used by default.
+
+* `--retry-interval-max`: The exponential backoff maximum value. 5 minutes is used by default.
+
 * `--workers <num>`: Number of simultaneously running `ControllerExpandVolume` operations. Default value is `10`.
 
 * `--metrics-address`: The TCP network address where the prometheus metrics endpoint will run (example: `:8080` which corresponds to port 8080 on local host). The default is empty string, which means metrics endpoint is disabled.

cmd/csi-resizer/main.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"k8s.io/client-go/util/workqueue"
 	"os"
 	"time"
 
@@ -43,6 +44,9 @@ var (
 	csiTimeout  = flag.Duration("csiTimeout", 15*time.Second, "Timeout for waiting for CSI driver socket.")
 	showVersion = flag.Bool("version", false, "Show version")
 
+	retryIntervalStart = flag.Duration("retry-interval-start", time.Second, "Initial retry interval of failed volume resize. It exponentially increases with each failure, up to retry-interval-max.")
+	retryIntervalMax   = flag.Duration("retry-interval-max", 5*time.Minute, "Maximum retry interval of failed volume resize.")
+
 	enableLeaderElection    = flag.Bool("leader-election", false, "Enable leader election.")
 	leaderElectionNamespace = flag.String("leader-election-namespace", "", "Namespace where the leader election resource lives. Defaults to the pod namespace if not set.")
 
@@ -82,7 +86,9 @@ func main() {
 	}
 
 	resizerName := csiResizer.Name()
-	rc := controller.NewResizeController(resizerName, csiResizer, kubeClient, *resyncPeriod, informerFactory)
+	rc := controller.NewResizeController(resizerName, csiResizer, kubeClient, *resyncPeriod, informerFactory,
+		workqueue.NewItemExponentialFailureRateLimiter(*retryIntervalStart, *retryIntervalMax),
+	)
 	run := func(ctx context.Context) {
 		informerFactory.Start(wait.NeverStop)
 		rc.Run(*workers, ctx)

pkg/controller/controller.go

@@ -65,7 +65,8 @@ func NewResizeController(
 	resizer resizer.Resizer,
 	kubeClient kubernetes.Interface,
 	resyncPeriod time.Duration,
-	informerFactory informers.SharedInformerFactory) ResizeController {
+	informerFactory informers.SharedInformerFactory,
+	pvcRateLimiter workqueue.RateLimiter) ResizeController {
 	pvInformer := informerFactory.Core().V1().PersistentVolumes()
 	pvcInformer := informerFactory.Core().V1().PersistentVolumeClaims()
 
@@ -76,7 +77,7 @@ func NewResizeController(
 		v1.EventSource{Component: fmt.Sprintf("external-resizer %s", name)})
 
 	claimQueue := workqueue.NewNamedRateLimitingQueue(
-		workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("%s-pvc", name))
+		pvcRateLimiter, fmt.Sprintf("%s-pvc", name))
 
 	ctrl := &resizeController{
 		name:          name,

pkg/controller/controller_test.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"context"
 	"fmt"
+	"k8s.io/client-go/util/workqueue"
 	"testing"
 	"time"
 
@@ -123,7 +124,7 @@ func TestController(t *testing.T) {
 			t.Fatalf("Test %s: Unable to create resizer: %v", test.Name, err)
 		}
 
-		controller := NewResizeController(driverName, csiResizer, kubeClient, time.Second, informerFactory)
+		controller := NewResizeController(driverName, csiResizer, kubeClient, time.Second, informerFactory, workqueue.DefaultControllerRateLimiter())
 		err = controller.(*resizeController).syncPVC(fmt.Sprintf("%s/%s", test.PVC.Namespace, test.PVC.Name))
 		if err != nil {
 			t.Fatalf("Test %s: Unexpected error: %v", test.Name, err)