Skip to content

Commit 9898208

Browse files
soxofaansoxofaan
committed
Automatically retry on 429 Too Many Requests
refs #441, #764
1 parent c6bc1c9 commit 9898208

File tree

3 files changed

+250
-1
lines changed

3 files changed

+250
-1
lines changed

‎openeo/rest/_connection.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from openeo.rest import OpenEoApiError, OpenEoApiPlainError, OpenEoRestError
1313
from openeo.rest.auth.auth import NullAuth
1414
from openeo.util import ContextTimer, ensure_list, str_truncate, url_join
15+
from openeo.utils.http import session_with_retries
1516

1617
_log = logging.getLogger(__name__)
1718

@@ -26,6 +27,7 @@ class RestApiConnection:
2627
def __init__(
2728
self,
2829
root_url: str,
30+
*,
2931
auth: Optional[AuthBase] = None,
3032
session: Optional[requests.Session] = None,
3133
default_timeout: Optional[int] = None,
@@ -34,7 +36,7 @@ def __init__(
3436
self._root_url = root_url
3537
self._auth = None
3638
self.auth = auth or NullAuth()
37-
self.session = session or requests.Session()
39+
self.session = session or session_with_retries()
3840
self.default_timeout = default_timeout or DEFAULT_TIMEOUT
3941
self.default_headers = {
4042
"User-Agent": "openeo-python-client/{cv} {py}/{pv} {pl}".format(

‎openeo/utils/http.py

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
"""
2+
openEO-oriented HTTP utilities
3+
"""
4+
5+
from typing import Collection, Union
6+
7+
import requests
8+
import requests.adapters
9+
10+
DEFAULT_RETRIES_TOTAL = 5
11+
12+
# On `backoff_factor`: it influences how much to sleep according to the formula:
13+
# sleep = {backoff factor} * (2 ** ({consecutive errors - 1}))
14+
# The sleep before the first retry will be skipped however.
15+
# For example with backoff_factor=2.5, the sleeps between consecutive attempts would be:
16+
# 0, 5, 10, 20, 40, ...
17+
DEFAULT_BACKOFF_FACTOR = 2.5
18+
19+
20+
DEFAULT_RETRY_FORCELIST = frozenset(
21+
[
22+
429, # Too Many Requests
23+
500, # Internal Server Error
24+
502, # Bad Gateway
25+
503, # Service Unavailable
26+
504, # Gateway Timeout
27+
]
28+
)
29+
30+
31+
def retry_adapter(
32+
*,
33+
total: int = DEFAULT_RETRIES_TOTAL,
34+
backoff_factor: float = DEFAULT_BACKOFF_FACTOR,
35+
status_forcelist: Collection[int] = DEFAULT_RETRY_FORCELIST,
36+
**kwargs,
37+
) -> requests.adapters.HTTPAdapter:
38+
"""
39+
Factory for creating a `requests.adapters.HTTPAdapter` with
40+
openEO-oriented retry settings.
41+
42+
:param total: Total number of retries to allow
43+
:param backoff_factor: scaling factor for sleeps between retries
44+
:param status_forcelist: A set of integer HTTP status codes that we should force a retry on.
45+
:param kwargs: additional kwargs to pass to `requests.adapters.Retry`
46+
:return:
47+
48+
Inspiration and references:
49+
- https://requests.readthedocs.io/en/latest/api/#requests.adapters.HTTPAdapter
50+
- https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry
51+
- https://findwork.dev/blog/advanced-usage-python-requests-timeouts-retries-hooks/#retry-on-failure
52+
"""
53+
retry = requests.adapters.Retry(
54+
total=total,
55+
backoff_factor=backoff_factor,
56+
status_forcelist=status_forcelist,
57+
**kwargs,
58+
)
59+
return requests.adapters.HTTPAdapter(max_retries=retry)
60+
61+
62+
def _to_retry_adapter(
63+
retries: Union[requests.adapters.HTTPAdapter, requests.adapters.Retry, dict, None]
64+
) -> requests.adapters.HTTPAdapter:
65+
"""
66+
Convert a retry specification to a mountable `requests.adapters.HTTPAdapter`.
67+
"""
68+
if isinstance(retries, requests.adapters.HTTPAdapter):
69+
adapter = retries
70+
elif isinstance(retries, requests.adapters.Retry):
71+
adapter = requests.adapters.HTTPAdapter(max_retries=retries)
72+
elif isinstance(retries, dict):
73+
adapter = retry_adapter(**retries)
74+
elif retries is None:
75+
adapter = retry_adapter()
76+
else:
77+
raise TypeError(f"Invalid type for retries: {type(retries)}")
78+
return adapter
79+
80+
81+
def session_with_retries(
82+
retries: Union[requests.adapters.HTTPAdapter, requests.adapters.Retry, dict, None] = None,
83+
) -> requests.Session:
84+
"""
85+
Factory for a requests session with openEO-oriented retry settings.
86+
87+
:param retries: The retry configuration, can be specified as:
88+
- :py:class:`requests.adapters.HTTPAdapter`
89+
- :py:class:`requests.adapters.Retry`
90+
- a dictionary with :py:class:`requests.adapters.Retry` arguments,
91+
e.g. ``total``, ``backoff_factor``, ``status_forcelist``, ...
92+
- ``None`` for default openEO-oriented retry settings
93+
"""
94+
session = requests.Session()
95+
adapter = _to_retry_adapter(retries=retries)
96+
session.mount("http://", adapter)
97+
session.mount("https://", adapter)
98+
return session

‎tests/utils/test_http.py

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
import contextlib
2+
import logging
3+
from typing import Iterator
4+
from unittest import mock
5+
6+
import httpretty
7+
import pytest
8+
import requests
9+
10+
from openeo.utils.http import session_with_retries
11+
12+
13+
class TestSessionWithRetries:
14+
@pytest.fixture(autouse=True)
15+
def time_sleep(self) -> Iterator[mock.Mock]:
16+
with mock.patch("time.sleep") as mock_sleep:
17+
yield mock_sleep
18+
19+
@pytest.fixture(autouse=True)
20+
def _auto_httpretty_enabled(self):
21+
"""Automatically activate httpretty for all tests in this class."""
22+
with httpretty.enabled(allow_net_connect=False):
23+
yield
24+
25+
def test_default_basic(self, time_sleep):
26+
responses = [
27+
httpretty.Response(status=429, body="Stop it!"),
28+
httpretty.Response(status=200, body="ok then"),
29+
]
30+
httpretty.register_uri(httpretty.GET, uri="https://example.test/", responses=responses)
31+
session = session_with_retries()
32+
resp = session.get("https://example.test/")
33+
assert resp.status_code == 200
34+
assert resp.text == "ok then"
35+
# Single retry does not trigger a sleep
36+
assert time_sleep.call_args_list == []
37+
38+
@pytest.mark.parametrize(
39+
["fail_count", "expected_sleeps", "success"],
40+
[
41+
(0, [], True),
42+
(1, [], True),
43+
(2, [5], True),
44+
(3, [5, 10], True),
45+
(5, [5, 10, 20, 40], True),
46+
(6, [5, 10, 20, 40], False),
47+
],
48+
)
49+
def test_default_multiple_attempts(self, time_sleep, fail_count, expected_sleeps, success):
50+
responses = [httpretty.Response(status=429, body="Stop it!")] * fail_count
51+
responses.append(httpretty.Response(status=200, body="ok then"))
52+
httpretty.register_uri(httpretty.GET, uri="https://example.test/", responses=responses)
53+
session = session_with_retries()
54+
55+
try:
56+
result = session.get("https://example.test/")
57+
except Exception as e:
58+
result = e
59+
60+
if success:
61+
assert isinstance(result, requests.Response)
62+
assert result.status_code == 200
63+
assert result.text == "ok then"
64+
else:
65+
assert isinstance(result, requests.exceptions.RetryError)
66+
67+
assert time_sleep.call_args_list == [mock.call(s) for s in expected_sleeps]
68+
69+
@pytest.mark.parametrize(
70+
["retry_config", "responses", "expected", "expected_sleeps"],
71+
[
72+
(
73+
# Fail with 500, which is in status_forcelist -> keep trying
74+
{"total": 3, "backoff_factor": 1.1, "status_forcelist": [500, 502, 503]},
75+
[
76+
httpretty.Response(status=500, body="Internal Server Error"),
77+
httpretty.Response(status=500, body="Internal Server Error"),
78+
httpretty.Response(status=500, body="Internal Server Error"),
79+
],
80+
requests.exceptions.RetryError,
81+
[2.2, 4.4],
82+
),
83+
(
84+
# Fail with 500, not in status_forcelist -> no retrying
85+
{"total": 3, "backoff_factor": 1.1, "status_forcelist": [502, 503]},
86+
[
87+
httpretty.Response(status=500, body="Internal Server Error"),
88+
httpretty.Response(status=500, body="Internal Server Error"),
89+
httpretty.Response(status=500, body="Internal Server Error"),
90+
],
91+
(500, "Internal Server Error"),
92+
[],
93+
),
94+
(
95+
# Multiple statuses in status_forcelist, retry until success
96+
{"total": 3, "backoff_factor": 1.1, "status_forcelist": [500, 502, 503]},
97+
[
98+
httpretty.Response(status=500, body="Internal Server Error"),
99+
httpretty.Response(status=502, body="Bad Gateway"),
100+
httpretty.Response(status=503, body="Service Unavailable"),
101+
httpretty.Response(status=200, body="Ok then"),
102+
],
103+
(200, "Ok then"),
104+
[2.2, 4.4],
105+
),
106+
],
107+
)
108+
def test_custom_retries(self, time_sleep, retry_config, responses, expected, expected_sleeps):
109+
httpretty.register_uri(httpretty.GET, uri="https://example.test/", responses=responses)
110+
session = session_with_retries(retries=retry_config)
111+
112+
try:
113+
result = session.get("https://example.test/")
114+
except Exception as e:
115+
result = e
116+
117+
if isinstance(expected, type):
118+
assert isinstance(result, expected)
119+
elif isinstance(expected, tuple):
120+
assert (result.status_code, result.text) == expected
121+
else:
122+
raise ValueError(expected)
123+
124+
assert time_sleep.call_args_list == [mock.call(s) for s in expected_sleeps]
125+
126+
return
127+
128+
@pytest.mark.parametrize(
129+
["retry_config"],
130+
[
131+
(None,),
132+
# Retry-after is even honored when 429 is not in status_forcelist
133+
({"status_forcelist": []},),
134+
],
135+
)
136+
def test_retry_after(self, time_sleep, retry_config):
137+
"""
138+
Test that the Retry-After header is respected.
139+
"""
140+
responses = [
141+
httpretty.Response(status=429, body="Stop it!", adding_headers={"Retry-After": "23"}),
142+
httpretty.Response(status=200, body="ok then"),
143+
]
144+
httpretty.register_uri(httpretty.GET, uri="https://example.test/", responses=responses)
145+
session = session_with_retries(retry_config)
146+
resp = session.get("https://example.test/")
147+
assert resp.status_code == 200
148+
assert resp.text == "ok then"
149+
assert time_sleep.call_args_list == [mock.call(23)]

0 commit comments

Comments
 (0)