1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
|
############################ Copyrights and license ############################
# #
# Copyright 2023 Enrico Minack <github@enrico.minack.dev> #
# Copyright 2023 Jirka Borovec <6035284+Borda@users.noreply.github.com> #
# Copyright 2023 Patryk Szulczyk <therealsoulcheck@gmail.com> #
# Copyright 2023 Trim21 <trim21.me@gmail.com> #
# Copyright 2024 Austin Sasko <austintyler0239@yahoo.com> #
# Copyright 2024 Enrico Minack <github@enrico.minack.dev> #
# Copyright 2024 Jirka Borovec <6035284+Borda@users.noreply.github.com> #
# #
# This file is part of PyGithub. #
# http://pygithub.readthedocs.io/ #
# #
# PyGithub is free software: you can redistribute it and/or modify it under #
# the terms of the GNU Lesser General Public License as published by the Free #
# Software Foundation, either version 3 of the License, or (at your option) #
# any later version. #
# #
# PyGithub is distributed in the hope that it will be useful, but WITHOUT ANY #
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS #
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more #
# details. #
# #
# You should have received a copy of the GNU Lesser General Public License #
# along with PyGithub. If not, see <http://www.gnu.org/licenses/>. #
# #
################################################################################
import json
import logging
from datetime import datetime, timezone
from logging import Logger
from types import TracebackType
from typing import Any, Optional
from requests import Response
from requests.models import CaseInsensitiveDict
from requests.utils import get_encoding_from_headers
from typing_extensions import Self
from urllib3 import Retry
from urllib3.connectionpool import ConnectionPool
from urllib3.exceptions import MaxRetryError
from urllib3.response import HTTPResponse
from github.GithubException import GithubException
from github.Requester import Requester
DEFAULT_SECONDARY_RATE_WAIT: int = 60
class GithubRetry(Retry):
"""
A Github-specific implementation of `urllib3.Retry`
This retries 403 responses if they are retry-able. Github requests are retry-able when
the response provides a `"Retry-After"` header, or the content indicates a rate limit error.
By default, response codes 403, and 500 up to 599 are retried. This can be configured
via the `status_forcelist` argument.
By default, all methods defined in `Retry.DEFAULT_ALLOWED_METHODS` are retried, plus GET and POST.
This can be configured via the `allowed_methods` argument.
"""
__logger: Optional[Logger] = None
# used to mock datetime, mock.patch("github.GithubRetry.date") does not work as this
# references the class, not the module (due to re-exporting in github/__init__.py)
__datetime = datetime
def __init__(self, secondary_rate_wait: float = DEFAULT_SECONDARY_RATE_WAIT, **kwargs: Any) -> None:
"""
:param secondary_rate_wait: seconds to wait before retrying secondary rate limit errors
:param kwargs: see urllib3.Retry for more arguments
"""
self.secondary_rate_wait = secondary_rate_wait
# 403 is too broad to be retried, but GitHub API signals rate limits via 403
# we retry 403 and look into the response header via Retry.increment
# to determine if we really retry that 403
kwargs["status_forcelist"] = kwargs.get("status_forcelist", list(range(500, 600))) + [403]
kwargs["allowed_methods"] = kwargs.get("allowed_methods", Retry.DEFAULT_ALLOWED_METHODS.union({"GET", "POST"}))
super().__init__(**kwargs)
def new(self, **kw: Any) -> Self:
kw.update(dict(secondary_rate_wait=self.secondary_rate_wait))
return super().new(**kw) # type: ignore
def increment( # type: ignore[override]
self,
method: Optional[str] = None,
url: Optional[str] = None,
response: Optional[HTTPResponse] = None, # type: ignore[override]
error: Optional[Exception] = None,
_pool: Optional[ConnectionPool] = None,
_stacktrace: Optional[TracebackType] = None,
) -> Retry:
if response:
# we retry 403 only when there is a Retry-After header (indicating it is retry-able)
# or the body message does imply a rate limit error
if response.status == 403:
self.__log(
logging.INFO,
f"Request {method} {url} failed with {response.status}: {response.reason}",
)
if "Retry-After" in response.headers:
# Sleeping 'Retry-After' seconds is implemented in urllib3.Retry.sleep() and called by urllib3
self.__log(
logging.INFO,
f'Retrying after {response.headers.get("Retry-After")} seconds',
)
else:
content = response.reason
# to identify retry-able methods, we inspect the response body
try:
content = self.get_content(response, url) # type: ignore
content = json.loads(content) # type: ignore
message = content.get("message") # type: ignore
except Exception as e:
# we want to fall back to the actual github exception (probably a rate limit error)
# but provide some context why we could not deal with it without another exception
try:
raise RuntimeError("Failed to inspect response message") from e
except RuntimeError as e:
raise GithubException(response.status, content, response.headers) from e # type: ignore
try:
if Requester.isRateLimitError(message):
rate_type = "primary" if Requester.isPrimaryRateLimitError(message) else "secondary"
self.__log(
logging.DEBUG,
f"Response body indicates retry-able {rate_type} rate limit error: {message}",
)
# check early that we are retrying at all
retry = super().increment(method, url, response, error, _pool, _stacktrace)
# we backoff primary rate limit at least until X-RateLimit-Reset,
# we backoff secondary rate limit at for secondary_rate_wait seconds
backoff = 0.0
if Requester.isPrimaryRateLimitError(message):
if "X-RateLimit-Reset" in response.headers:
value = response.headers.get("X-RateLimit-Reset")
if value and value.isdigit():
reset = self.__datetime.fromtimestamp(int(value), timezone.utc)
delta = reset - self.__datetime.now(timezone.utc)
resetBackoff = delta.total_seconds()
if resetBackoff > 0:
self.__log(
logging.DEBUG,
f"Reset occurs in {str(delta)} ({value} / {reset})",
)
# plus 1s as it is not clear when in that second the reset occurs
backoff = resetBackoff + 1
else:
backoff = self.secondary_rate_wait
# we backoff at least retry's next backoff
retry_backoff = retry.get_backoff_time()
if retry_backoff > backoff:
if backoff > 0:
self.__log(
logging.DEBUG,
f"Retry backoff of {retry_backoff}s exceeds "
f"required rate limit backoff of {backoff}s".replace(".0s", "s"),
)
backoff = retry_backoff
def get_backoff_time() -> float:
return backoff
self.__log(
logging.INFO,
f"Setting next backoff to {backoff}s".replace(".0s", "s"),
)
retry.get_backoff_time = get_backoff_time # type: ignore
return retry
self.__log(
logging.DEBUG,
"Response message does not indicate retry-able error",
)
raise Requester.createException(response.status, response.headers, content) # type: ignore
except (MaxRetryError, GithubException):
raise
except Exception as e:
# we want to fall back to the actual github exception (probably a rate limit error)
# but provide some context why we could not deal with it without another exception
try:
raise RuntimeError("Failed to determine retry backoff") from e
except RuntimeError as e:
raise GithubException(response.status, content, response.headers) from e # type: ignore
raise GithubException(
response.status, # type: ignore
content, # type: ignore
response.headers, # type: ignore
) # type: ignore
# retry the request as usual
return super().increment(method, url, response, error, _pool, _stacktrace)
@staticmethod
def get_content(resp: HTTPResponse, url: str) -> bytes: # type: ignore[override]
# logic taken from HTTPAdapter.build_response (requests.adapters)
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, "status", None) # type: ignore
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason # type: ignore
response.url = url
return response.content
def __log(self, level: int, message: str, **kwargs: Any) -> None:
if self.__logger is None:
self.__logger = logging.getLogger(__name__)
if self.__logger.isEnabledFor(level):
self.__logger.log(level, message, **kwargs)
|
