From 5389fb0489a8fed4a76f5dbc18f0218768709ce3 Mon Sep 17 00:00:00 2001 From: Luca Cillario Date: Mon, 14 Nov 2022 14:09:57 +0100 Subject: [PATCH 01/10] Retry documentation. (#2166) --- docs/backoff.rst | 2 ++ docs/exceptions.rst | 2 +- docs/retry.rst | 67 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/docs/backoff.rst b/docs/backoff.rst index e640b5682e..c5ab01ab03 100644 --- a/docs/backoff.rst +++ b/docs/backoff.rst @@ -1,3 +1,5 @@ +.. _backoff-label: + Backoff ############# diff --git a/docs/exceptions.rst b/docs/exceptions.rst index b8aeb33e49..8a9fe457fb 100644 --- a/docs/exceptions.rst +++ b/docs/exceptions.rst @@ -1,4 +1,4 @@ - +.. _exceptions-label: Exceptions ########## diff --git a/docs/retry.rst b/docs/retry.rst index 2b4f22c2f6..6b574e8653 100644 --- a/docs/retry.rst +++ b/docs/retry.rst @@ -2,4 +2,69 @@ Retry Helpers ############# .. automodule:: redis.retry - :members: \ No newline at end of file + :members: + + +Retry in Redis Standalone +************************** + +>>> from redis.backoff import ExponentialBackoff +>>> from redis.retry import Retry +>>> from redis.client import Redis +>>> from redis.exceptions import ( +>>> BusyLoadingError, +>>> ConnectionError, +>>> TimeoutError +>>> ) +>>> +>>> # Run 3 retries with exponential backoff strategy +>>> retry = Retry(ExponentialBackoff(), 3) +>>> # Redis client with retries on custom errors +>>> r = Redis(host='localhost', port=6379, retry=retry, retry_on_error=[BusyLoadingError, ConnectionError, TimeoutError]) +>>> # Redis client with retries on TimeoutError only +>>> r_only_timeout = Redis(host='localhost', port=6379, retry=retry, retry_on_timeout=True) + +As you can see from the example above, Redis client supports 3 parameters to configure the retry behaviour: + +* ``retry``: :class:`~.Retry` instance with a :ref:`backoff-label` strategy and the max number of retries +* ``retry_on_error``: list of :ref:`exceptions-label` to retry on +* ``retry_on_timeout``: if ``True``, retry on :class:`~.TimeoutError` only + +If either ``retry_on_error`` or ``retry_on_timeout`` are passed and no ``retry`` is given, +by default it uses a ``Retry(NoBackoff(), 1)`` (meaning 1 retry right after the first failure). + + +Retry in Redis Cluster +************************** + +>>> from redis.backoff import ExponentialBackoff +>>> from redis.retry import Retry +>>> from redis.cluster import RedisCluster +>>> +>>> # Run 3 retries with exponential backoff strategy +>>> retry = Retry(ExponentialBackoff(), 1) +>>> # Redis Cluster client with retries +>>> rc = RedisCluster(host='localhost', port=6379, retry=retry, cluster_error_retry_attempts=2) + +Retry behaviour in Redis Cluster is a little bit different from Standalone: + +* ``retry``: :class:`~.Retry` instance with a :ref:`backoff-label` strategy and the max number of retries, default value is ``Retry(NoBackoff(), 0)`` +* ``cluster_error_retry_attempts``: number of times to retry before raising an error when :class:`~.TimeoutError` or :class:`~.ConnectionError` or :class:`~.ClusterDownError` are encountered, default value is ``3`` + +Let's consider the following example: + +>>> from redis.backoff import ExponentialBackoff +>>> from redis.retry import Retry +>>> from redis.cluster import RedisCluster +>>> +>>> rc = RedisCluster(host='localhost', port=6379, retry=Retry(ExponentialBackoff(), 6), cluster_error_retry_attempts=1) +>>> rc.set('foo', 'bar') + +#. the client library calculates the hash slot for key 'foo' +#. given the hash slot it then determines which node to connect to, in order to execute the command +#. during the connection a :class:`~.ConnectionError` is raised +#. because we set ``retry=Retry(ExponentialBackoff(), 6)``, the client will try to reconnect to the node up to 6 times, with an exponential backoff between each attempt +#. even after 6 retries, the client is still unable to connect +#. because we set ``cluster_error_retry_attempts=1``, before giving up, the client starts a cluster update, it removes the failed node from the startup nodes and re-initializes the cluster +#. after the cluster has been re-initialized, it starts a new cycle of retries, up to 6 retries, with an exponential backoff +#. if the client can connect, we're good. Otherwise the exception if finally raised to the caller, because we run out of attempts \ No newline at end of file From 81fd500106f9f3b9090f8a43cc1a7c0edbbcbf3d Mon Sep 17 00:00:00 2001 From: Luca Cillario Date: Mon, 14 Nov 2022 14:19:21 +0100 Subject: [PATCH 02/10] Fixed typo. --- docs/retry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/retry.rst b/docs/retry.rst index 6b574e8653..78fbd27a63 100644 --- a/docs/retry.rst +++ b/docs/retry.rst @@ -42,7 +42,7 @@ Retry in Redis Cluster >>> from redis.cluster import RedisCluster >>> >>> # Run 3 retries with exponential backoff strategy ->>> retry = Retry(ExponentialBackoff(), 1) +>>> retry = Retry(ExponentialBackoff(), 3) >>> # Redis Cluster client with retries >>> rc = RedisCluster(host='localhost', port=6379, retry=retry, cluster_error_retry_attempts=2) From 7d8723a732cd5ef68fa3068ca05670304fcf6115 Mon Sep 17 00:00:00 2001 From: Luca Cillario Date: Sun, 20 Nov 2022 11:36:24 +0100 Subject: [PATCH 03/10] Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> --- docs/retry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/retry.rst b/docs/retry.rst index 78fbd27a63..7ba4993f3e 100644 --- a/docs/retry.rst +++ b/docs/retry.rst @@ -61,7 +61,7 @@ Let's consider the following example: >>> rc.set('foo', 'bar') #. the client library calculates the hash slot for key 'foo' -#. given the hash slot it then determines which node to connect to, in order to execute the command +#. given the hash slot, it then determines which node to connect to, in order to execute the command. #. during the connection a :class:`~.ConnectionError` is raised #. because we set ``retry=Retry(ExponentialBackoff(), 6)``, the client will try to reconnect to the node up to 6 times, with an exponential backoff between each attempt #. even after 6 retries, the client is still unable to connect From 1ae9dd2e95d7da1d95eebfd22ae3326188dc7121 Mon Sep 17 00:00:00 2001 From: Luca Cillario Date: Sun, 20 Nov 2022 11:36:32 +0100 Subject: [PATCH 04/10] Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> --- docs/retry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/retry.rst b/docs/retry.rst index 7ba4993f3e..7039375105 100644 --- a/docs/retry.rst +++ b/docs/retry.rst @@ -60,7 +60,7 @@ Let's consider the following example: >>> rc = RedisCluster(host='localhost', port=6379, retry=Retry(ExponentialBackoff(), 6), cluster_error_retry_attempts=1) >>> rc.set('foo', 'bar') -#. the client library calculates the hash slot for key 'foo' +#. the client library calculates the hash slot for key 'foo'. #. given the hash slot, it then determines which node to connect to, in order to execute the command. #. during the connection a :class:`~.ConnectionError` is raised #. because we set ``retry=Retry(ExponentialBackoff(), 6)``, the client will try to reconnect to the node up to 6 times, with an exponential backoff between each attempt From 7d3e42e2dbd67f9c7a1533e40f400d5a14930c12 Mon Sep 17 00:00:00 2001 From: Luca Cillario Date: Sun, 20 Nov 2022 11:36:37 +0100 Subject: [PATCH 05/10] Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> --- docs/retry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/retry.rst b/docs/retry.rst index 7039375105..fbcc6891b8 100644 --- a/docs/retry.rst +++ b/docs/retry.rst @@ -62,7 +62,7 @@ Let's consider the following example: #. the client library calculates the hash slot for key 'foo'. #. given the hash slot, it then determines which node to connect to, in order to execute the command. -#. during the connection a :class:`~.ConnectionError` is raised +#. during the connection, a :class:`~.ConnectionError` is raised. #. because we set ``retry=Retry(ExponentialBackoff(), 6)``, the client will try to reconnect to the node up to 6 times, with an exponential backoff between each attempt #. even after 6 retries, the client is still unable to connect #. because we set ``cluster_error_retry_attempts=1``, before giving up, the client starts a cluster update, it removes the failed node from the startup nodes and re-initializes the cluster From b2801bb819980ee122d9067591adf3cd72234543 Mon Sep 17 00:00:00 2001 From: Luca Cillario Date: Sun, 20 Nov 2022 11:36:44 +0100 Subject: [PATCH 06/10] Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> --- docs/retry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/retry.rst b/docs/retry.rst index fbcc6891b8..034cc4acf9 100644 --- a/docs/retry.rst +++ b/docs/retry.rst @@ -63,7 +63,7 @@ Let's consider the following example: #. the client library calculates the hash slot for key 'foo'. #. given the hash slot, it then determines which node to connect to, in order to execute the command. #. during the connection, a :class:`~.ConnectionError` is raised. -#. because we set ``retry=Retry(ExponentialBackoff(), 6)``, the client will try to reconnect to the node up to 6 times, with an exponential backoff between each attempt +#. because we set ``retry=Retry(ExponentialBackoff(), 6)``, the client tries to reconnect to the node up to 6 times, with an exponential backoff between each attempt. #. even after 6 retries, the client is still unable to connect #. because we set ``cluster_error_retry_attempts=1``, before giving up, the client starts a cluster update, it removes the failed node from the startup nodes and re-initializes the cluster #. after the cluster has been re-initialized, it starts a new cycle of retries, up to 6 retries, with an exponential backoff From 65dc5171990a918c871c4cb07ea803c2978efa0c Mon Sep 17 00:00:00 2001 From: Luca Cillario Date: Sun, 20 Nov 2022 11:36:49 +0100 Subject: [PATCH 07/10] Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> --- docs/retry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/retry.rst b/docs/retry.rst index 034cc4acf9..9575103cbe 100644 --- a/docs/retry.rst +++ b/docs/retry.rst @@ -65,6 +65,6 @@ Let's consider the following example: #. during the connection, a :class:`~.ConnectionError` is raised. #. because we set ``retry=Retry(ExponentialBackoff(), 6)``, the client tries to reconnect to the node up to 6 times, with an exponential backoff between each attempt. #. even after 6 retries, the client is still unable to connect -#. because we set ``cluster_error_retry_attempts=1``, before giving up, the client starts a cluster update, it removes the failed node from the startup nodes and re-initializes the cluster +#. because we set ``cluster_error_retry_attempts=1``, before giving up, the client starts a cluster update, removes the failed node from the startup nodes, and re-initializes the cluster. #. after the cluster has been re-initialized, it starts a new cycle of retries, up to 6 retries, with an exponential backoff #. if the client can connect, we're good. Otherwise the exception if finally raised to the caller, because we run out of attempts \ No newline at end of file From 1e04bd02926db5ebd54b03f38164bbb7ba022e79 Mon Sep 17 00:00:00 2001 From: Luca Cillario Date: Sun, 20 Nov 2022 11:36:55 +0100 Subject: [PATCH 08/10] Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> --- docs/retry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/retry.rst b/docs/retry.rst index 9575103cbe..fb928de46a 100644 --- a/docs/retry.rst +++ b/docs/retry.rst @@ -64,7 +64,7 @@ Let's consider the following example: #. given the hash slot, it then determines which node to connect to, in order to execute the command. #. during the connection, a :class:`~.ConnectionError` is raised. #. because we set ``retry=Retry(ExponentialBackoff(), 6)``, the client tries to reconnect to the node up to 6 times, with an exponential backoff between each attempt. -#. even after 6 retries, the client is still unable to connect +#. even after 6 retries, the client is still unable to connect. #. because we set ``cluster_error_retry_attempts=1``, before giving up, the client starts a cluster update, removes the failed node from the startup nodes, and re-initializes the cluster. #. after the cluster has been re-initialized, it starts a new cycle of retries, up to 6 retries, with an exponential backoff #. if the client can connect, we're good. Otherwise the exception if finally raised to the caller, because we run out of attempts \ No newline at end of file From d716cd85398888e75e02991f1d5bab7f195f21b3 Mon Sep 17 00:00:00 2001 From: Luca Cillario Date: Sun, 20 Nov 2022 11:36:59 +0100 Subject: [PATCH 09/10] Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> --- docs/retry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/retry.rst b/docs/retry.rst index fb928de46a..e2b959718d 100644 --- a/docs/retry.rst +++ b/docs/retry.rst @@ -66,5 +66,5 @@ Let's consider the following example: #. because we set ``retry=Retry(ExponentialBackoff(), 6)``, the client tries to reconnect to the node up to 6 times, with an exponential backoff between each attempt. #. even after 6 retries, the client is still unable to connect. #. because we set ``cluster_error_retry_attempts=1``, before giving up, the client starts a cluster update, removes the failed node from the startup nodes, and re-initializes the cluster. -#. after the cluster has been re-initialized, it starts a new cycle of retries, up to 6 retries, with an exponential backoff +#. after the cluster has been re-initialized, it starts a new cycle of retries, up to 6 retries, with an exponential backoff. #. if the client can connect, we're good. Otherwise the exception if finally raised to the caller, because we run out of attempts \ No newline at end of file From 90e9abfc643bd547b326db41cc70871e217b5424 Mon Sep 17 00:00:00 2001 From: Luca Cillario Date: Sun, 20 Nov 2022 11:37:06 +0100 Subject: [PATCH 10/10] Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> --- docs/retry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/retry.rst b/docs/retry.rst index e2b959718d..acf198ec94 100644 --- a/docs/retry.rst +++ b/docs/retry.rst @@ -67,4 +67,4 @@ Let's consider the following example: #. even after 6 retries, the client is still unable to connect. #. because we set ``cluster_error_retry_attempts=1``, before giving up, the client starts a cluster update, removes the failed node from the startup nodes, and re-initializes the cluster. #. after the cluster has been re-initialized, it starts a new cycle of retries, up to 6 retries, with an exponential backoff. -#. if the client can connect, we're good. Otherwise the exception if finally raised to the caller, because we run out of attempts \ No newline at end of file +#. if the client can connect, we're good. Otherwise, the exception is finally raised to the caller, because we've run out of attempts. \ No newline at end of file