File: timeout.py

package info (click to toggle)
python-google-api-core 2.25.1-1~exp1
  • links: PTS, VCS
  • area: main
  • in suites: experimental
  • size: 1,304 kB
  • sloc: python: 15,274; makefile: 3
file content (294 lines) | stat: -rw-r--r-- 10,279 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
# Copyright 2017 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Decorators for applying timeout arguments to functions.

These decorators are used to wrap API methods to apply either a
Deadline-dependent (recommended), constant (DEPRECATED) or exponential
(DEPRECATED) timeout argument.

For example, imagine an API method that can take a while to return results,
such as one that might block until a resource is ready:

.. code-block:: python

    def is_thing_ready(timeout=None):
        response = requests.get('https://example.com/is_thing_ready')
        response.raise_for_status()
        return response.json()

This module allows a function like this to be wrapped so that timeouts are
automatically determined, for example:

.. code-block:: python

    timeout_ = timeout.ExponentialTimeout()
    is_thing_ready_with_timeout = timeout_(is_thing_ready)

    for n in range(10):
        try:
            is_thing_ready_with_timeout({'example': 'data'})
        except:
            pass

In this example the first call to ``is_thing_ready`` will have a relatively
small timeout (like 1 second). If the resource is available and the request
completes quickly, the loop exits. But, if the resource isn't yet available
and the request times out, it'll be retried - this time with a larger timeout.

In the broader context these decorators are typically combined with
:mod:`google.api_core.retry` to implement API methods with a signature that
matches ``api_method(request, timeout=None, retry=None)``.
"""

from __future__ import unicode_literals

import datetime
import functools

from google.api_core import datetime_helpers

_DEFAULT_INITIAL_TIMEOUT = 5.0  # seconds
_DEFAULT_MAXIMUM_TIMEOUT = 30.0  # seconds
_DEFAULT_TIMEOUT_MULTIPLIER = 2.0
# If specified, must be in seconds. If none, deadline is not used in the
# timeout calculation.
_DEFAULT_DEADLINE = None


class TimeToDeadlineTimeout(object):
    """A decorator that decreases timeout set for an RPC based on how much time
    has left till its deadline. The deadline is calculated as
    ``now + initial_timeout`` when this decorator is first called for an rpc.

    In other words this decorator implements deadline semantics in terms of a
    sequence of decreasing timeouts t0 > t1 > t2 ... tn >= 0.

    Args:
        timeout (Optional[float]): the timeout (in seconds) to applied to the
            wrapped function. If `None`, the target function is expected to
            never timeout.
    """

    def __init__(self, timeout=None, clock=datetime_helpers.utcnow):
        self._timeout = timeout
        self._clock = clock

    def __call__(self, func):
        """Apply the timeout decorator.

        Args:
            func (Callable): The function to apply the timeout argument to.
                This function must accept a timeout keyword argument.

        Returns:
            Callable: The wrapped function.
        """

        first_attempt_timestamp = self._clock().timestamp()

        @functools.wraps(func)
        def func_with_timeout(*args, **kwargs):
            """Wrapped function that adds timeout."""

            if self._timeout is not None:
                # All calculations are in seconds
                now_timestamp = self._clock().timestamp()

                # To avoid usage of nonlocal but still have round timeout
                # numbers for first attempt (in most cases the only attempt made
                # for an RPC.
                if now_timestamp - first_attempt_timestamp < 0.001:
                    now_timestamp = first_attempt_timestamp

                time_since_first_attempt = now_timestamp - first_attempt_timestamp
                remaining_timeout = self._timeout - time_since_first_attempt

                # Although the `deadline` parameter in `google.api_core.retry.Retry`
                # is deprecated, and should be treated the same as the `timeout`,
                # it is still possible for the `deadline` argument in
                # `google.api_core.retry.Retry` to be larger than the `timeout`.
                # See https://github.com/googleapis/python-api-core/issues/654
                # Only positive non-zero timeouts are supported.
                # Revert back to the initial timeout for negative or 0 timeout values.
                if remaining_timeout < 1:
                    remaining_timeout = self._timeout

                kwargs["timeout"] = remaining_timeout

            return func(*args, **kwargs)

        return func_with_timeout

    def __str__(self):
        return "<TimeToDeadlineTimeout timeout={:.1f}>".format(self._timeout)


class ConstantTimeout(object):
    """A decorator that adds a constant timeout argument.

    DEPRECATED: use ``TimeToDeadlineTimeout`` instead.

    This is effectively equivalent to
    ``functools.partial(func, timeout=timeout)``.

    Args:
        timeout (Optional[float]): the timeout (in seconds) to applied to the
            wrapped function. If `None`, the target function is expected to
            never timeout.
    """

    def __init__(self, timeout=None):
        self._timeout = timeout

    def __call__(self, func):
        """Apply the timeout decorator.

        Args:
            func (Callable): The function to apply the timeout argument to.
                This function must accept a timeout keyword argument.

        Returns:
            Callable: The wrapped function.
        """

        @functools.wraps(func)
        def func_with_timeout(*args, **kwargs):
            """Wrapped function that adds timeout."""
            kwargs["timeout"] = self._timeout
            return func(*args, **kwargs)

        return func_with_timeout

    def __str__(self):
        return "<ConstantTimeout timeout={:.1f}>".format(self._timeout)


def _exponential_timeout_generator(initial, maximum, multiplier, deadline):
    """A generator that yields exponential timeout values.

    Args:
        initial (float): The initial timeout.
        maximum (float): The maximum timeout.
        multiplier (float): The multiplier applied to the timeout.
        deadline (float): The overall deadline across all invocations.

    Yields:
        float: A timeout value.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = datetime.datetime.max

    timeout = initial
    while True:
        now = datetime_helpers.utcnow()
        yield min(
            # The calculated timeout based on invocations.
            timeout,
            # The set maximum timeout.
            maximum,
            # The remaining time before the deadline is reached.
            float((deadline_datetime - now).seconds),
        )
        timeout = timeout * multiplier


class ExponentialTimeout(object):
    """A decorator that adds an exponentially increasing timeout argument.

    DEPRECATED: the concept of incrementing timeout exponentially has been
    deprecated. Use ``TimeToDeadlineTimeout`` instead.

    This is useful if a function is called multiple times. Each time the
    function is called this decorator will calculate a new timeout parameter
    based on the the number of times the function has been called.

    For example

    .. code-block:: python

    Args:
        initial (float): The initial timeout to pass.
        maximum (float): The maximum timeout for any one call.
        multiplier (float): The multiplier applied to the timeout for each
            invocation.
        deadline (Optional[float]): The overall deadline across all
            invocations. This is used to prevent a very large calculated
            timeout from pushing the overall execution time over the deadline.
            This is especially useful in conjunction with
            :mod:`google.api_core.retry`. If ``None``, the timeouts will not
            be adjusted to accommodate an overall deadline.
    """

    def __init__(
        self,
        initial=_DEFAULT_INITIAL_TIMEOUT,
        maximum=_DEFAULT_MAXIMUM_TIMEOUT,
        multiplier=_DEFAULT_TIMEOUT_MULTIPLIER,
        deadline=_DEFAULT_DEADLINE,
    ):
        self._initial = initial
        self._maximum = maximum
        self._multiplier = multiplier
        self._deadline = deadline

    def with_deadline(self, deadline):
        """Return a copy of this timeout with the given deadline.

        Args:
            deadline (float): The overall deadline across all invocations.

        Returns:
            ExponentialTimeout: A new instance with the given deadline.
        """
        return ExponentialTimeout(
            initial=self._initial,
            maximum=self._maximum,
            multiplier=self._multiplier,
            deadline=deadline,
        )

    def __call__(self, func):
        """Apply the timeout decorator.

        Args:
            func (Callable): The function to apply the timeout argument to.
                This function must accept a timeout keyword argument.

        Returns:
            Callable: The wrapped function.
        """
        timeouts = _exponential_timeout_generator(
            self._initial, self._maximum, self._multiplier, self._deadline
        )

        @functools.wraps(func)
        def func_with_timeout(*args, **kwargs):
            """Wrapped function that adds timeout."""
            kwargs["timeout"] = next(timeouts)
            return func(*args, **kwargs)

        return func_with_timeout

    def __str__(self):
        return (
            "<ExponentialTimeout initial={:.1f}, maximum={:.1f}, "
            "multiplier={:.1f}, deadline={:.1f}>".format(
                self._initial, self._maximum, self._multiplier, self._deadline
            )
        )