1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
|
#!/usr/bin/env python3
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Generic retry wrapper for Git operations.
This is largely DEPRECATED in favor of the Infra Git wrapper:
https://chromium.googlesource.com/infra/infra/+/HEAD/go/src/infra/tools/git
"""
import logging
import optparse
import os
import subprocess
import sys
import threading
import time
from git_common import GIT_EXE, GIT_TRANSIENT_ERRORS_RE
class TeeThread(threading.Thread):
def __init__(self, fd, out_fd, name):
super(TeeThread, self).__init__(name='git-retry.tee.%s' % (name, ))
self.data = None
self.fd = fd
self.out_fd = out_fd
def run(self):
chunks = []
for line in self.fd:
line = line.decode('utf-8')
chunks.append(line)
self.out_fd.write(line)
self.data = ''.join(chunks)
class GitRetry(object):
logger = logging.getLogger('git-retry')
DEFAULT_DELAY_SECS = 3.0
DEFAULT_RETRY_COUNT = 5
def __init__(self, retry_count=None, delay=None, delay_factor=None):
self.retry_count = retry_count or self.DEFAULT_RETRY_COUNT
self.delay = max(delay, 0) if delay else 0
self.delay_factor = max(delay_factor, 0) if delay_factor else 0
def shouldRetry(self, stderr):
m = GIT_TRANSIENT_ERRORS_RE.search(stderr)
if not m:
return False
self.logger.info("Encountered known transient error: [%s]",
stderr[m.start():m.end()])
return True
@staticmethod
def execute(*args):
args = (GIT_EXE, ) + args
proc = subprocess.Popen(
args,
stderr=subprocess.PIPE,
)
stderr_tee = TeeThread(proc.stderr, sys.stderr, 'stderr')
# Start our process. Collect/tee 'stdout' and 'stderr'.
stderr_tee.start()
try:
proc.wait()
except KeyboardInterrupt:
proc.kill()
raise
finally:
stderr_tee.join()
return proc.returncode, None, stderr_tee.data
def computeDelay(self, iteration):
"""Returns: the delay (in seconds) for a given iteration
The first iteration has a delay of '0'.
Args:
iteration: (int) The iteration index (starting with zero as the
first iteration)
"""
if (not self.delay) or (iteration == 0):
return 0
if self.delay_factor == 0:
# Linear delay
return iteration * self.delay
# Exponential delay
return (self.delay_factor**(iteration - 1)) * self.delay
def __call__(self, *args):
returncode = 0
for i in range(self.retry_count):
# If the previous run failed and a delay is configured, delay before
# the next run.
delay = self.computeDelay(i)
if delay > 0:
self.logger.info("Delaying for [%s second(s)] until next retry",
delay)
time.sleep(delay)
self.logger.debug("Executing subprocess (%d/%d) with arguments: %s",
(i + 1), self.retry_count, args)
returncode, _, stderr = self.execute(*args)
self.logger.debug("Process terminated with return code: %d",
returncode)
if returncode == 0:
break
if not self.shouldRetry(stderr):
self.logger.error(
"Process failure was not known to be transient; "
"terminating with return code %d", returncode)
break
return returncode
def main(args):
# If we're using the Infra Git wrapper, do nothing here.
# https://chromium.googlesource.com/infra/infra/+/HEAD/go/src/infra/tools/git
if 'INFRA_GIT_WRAPPER' in os.environ:
# Remove Git's execution path from PATH so that our call-through
# re-invokes the Git wrapper. See crbug.com/721450
env = os.environ.copy()
git_exec = subprocess.check_output([GIT_EXE, '--exec-path']).strip()
env['PATH'] = os.pathsep.join([
elem for elem in env.get('PATH', '').split(os.pathsep)
if elem != git_exec
])
return subprocess.call([GIT_EXE] + args, env=env)
parser = optparse.OptionParser()
parser.disable_interspersed_args()
parser.add_option(
'-v',
'--verbose',
action='count',
default=0,
help="Increase verbosity; can be specified multiple times")
parser.add_option('-c',
'--retry-count',
metavar='COUNT',
type=int,
default=GitRetry.DEFAULT_RETRY_COUNT,
help="Number of times to retry (default=%default)")
parser.add_option('-d',
'--delay',
metavar='SECONDS',
type=float,
default=GitRetry.DEFAULT_DELAY_SECS,
help="Specifies the amount of time (in seconds) to wait "
"between successive retries (default=%default). This "
"can be zero.")
parser.add_option(
'-D',
'--delay-factor',
metavar='FACTOR',
type=int,
default=2,
help="The exponential factor to apply to delays in between "
"successive failures (default=%default). If this is "
"zero, delays will increase linearly. Set this to "
"one to have a constant (non-increasing) delay.")
opts, args = parser.parse_args(args)
# Configure logging verbosity
if opts.verbose == 0:
logging.getLogger().setLevel(logging.WARNING)
elif opts.verbose == 1:
logging.getLogger().setLevel(logging.INFO)
else:
logging.getLogger().setLevel(logging.DEBUG)
# Execute retries
retry = GitRetry(
retry_count=opts.retry_count,
delay=opts.delay,
delay_factor=opts.delay_factor,
)
return retry(*args)
if __name__ == '__main__':
logging.basicConfig()
logging.getLogger().setLevel(logging.WARNING)
try:
sys.exit(main(sys.argv[2:]))
except KeyboardInterrupt:
sys.stderr.write('interrupted\n')
sys.exit(1)
|
