1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
|
#!/usr/bin/env python
import sys, os, glob, subprocess
from bup import options, git
from bup.helpers import *
par2_ok = 0
nullf = open('/dev/null')
def debug(s):
if opt.verbose:
log(s)
def run(argv):
# at least in python 2.5, using "stdout=2" or "stdout=sys.stderr" below
# doesn't actually work, because subprocess closes fd #2 right before
# execing for some reason. So we work around it by duplicating the fd
# first.
fd = os.dup(2) # copy stderr
try:
p = subprocess.Popen(argv, stdout=fd, close_fds=False)
return p.wait()
finally:
os.close(fd)
def par2_setup():
global par2_ok
rv = 1
try:
p = subprocess.Popen(['par2', '--help'],
stdout=nullf, stderr=nullf, stdin=nullf)
rv = p.wait()
except OSError:
log('fsck: warning: par2 not found; disabling recovery features.\n')
else:
par2_ok = 1
def parv(lvl):
if opt.verbose >= lvl:
if istty:
return []
else:
return ['-q']
else:
return ['-qq']
def par2_generate(base):
return run(['par2', 'create', '-n1', '-c200'] + parv(2)
+ ['--', base, base+'.pack', base+'.idx'])
def par2_verify(base):
return run(['par2', 'verify'] + parv(3) + ['--', base])
def par2_repair(base):
return run(['par2', 'repair'] + parv(2) + ['--', base])
def quick_verify(base):
f = open(base + '.pack', 'rb')
f.seek(-20, 2)
wantsum = f.read(20)
assert(len(wantsum) == 20)
f.seek(0)
sum = Sha1()
for b in chunkyreader(f, os.fstat(f.fileno()).st_size - 20):
sum.update(b)
if sum.digest() != wantsum:
raise ValueError('expected %r, got %r' % (wantsum.encode('hex'),
sum.hexdigest()))
def git_verify(base):
if opt.quick:
try:
quick_verify(base)
except Exception, e:
debug('error: %s\n' % e)
return 1
return 0
else:
return run(['git', 'verify-pack', '--', base])
def do_pack(base, last):
code = 0
if par2_ok and par2_exists and (opt.repair or not opt.generate):
vresult = par2_verify(base)
if vresult != 0:
if opt.repair:
rresult = par2_repair(base)
if rresult != 0:
print '%s par2 repair: failed (%d)' % (last, rresult)
code = rresult
else:
print '%s par2 repair: succeeded (0)' % last
code = 100
else:
print '%s par2 verify: failed (%d)' % (last, vresult)
code = vresult
else:
print '%s ok' % last
elif not opt.generate or (par2_ok and not par2_exists):
gresult = git_verify(base)
if gresult != 0:
print '%s git verify: failed (%d)' % (last, gresult)
code = gresult
else:
if par2_ok and opt.generate:
presult = par2_generate(base)
if presult != 0:
print '%s par2 create: failed (%d)' % (last, presult)
code = presult
else:
print '%s ok' % last
else:
print '%s ok' % last
else:
assert(opt.generate and (not par2_ok or par2_exists))
debug(' skipped: par2 file already generated.\n')
return code
optspec = """
bup fsck [options...] [filenames...]
--
r,repair attempt to repair errors using par2 (dangerous!)
g,generate generate auto-repair information using par2
v,verbose increase verbosity (can be used more than once)
quick just check pack sha1sum, don't use git verify-pack
j,jobs= run 'n' jobs in parallel
par2-ok immediately return 0 if par2 is ok, 1 if not
disable-par2 ignore par2 even if it is available
"""
o = options.Options('bup fsck', optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
par2_setup()
if opt.par2_ok:
if par2_ok:
sys.exit(0) # 'true' in sh
else:
sys.exit(1)
if opt.disable_par2:
par2_ok = 0
git.check_repo_or_die()
if not extra:
debug('fsck: No filenames given: checking all packs.\n')
extra = glob.glob(git.repo('objects/pack/*.pack'))
code = 0
count = 0
outstanding = {}
for name in extra:
if name.endswith('.pack'):
base = name[:-5]
elif name.endswith('.idx'):
base = name[:-4]
elif name.endswith('.par2'):
base = name[:-5]
elif os.path.exists(name + '.pack'):
base = name
else:
raise Exception('%s is not a pack file!' % name)
(dir,last) = os.path.split(base)
par2_exists = os.path.exists(base + '.par2')
if par2_exists and os.stat(base + '.par2').st_size == 0:
par2_exists = 0
sys.stdout.flush()
debug('fsck: checking %s (%s)\n'
% (last, par2_ok and par2_exists and 'par2' or 'git'))
if not opt.verbose:
progress('fsck (%d/%d)\r' % (count, len(extra)))
if not opt.jobs:
nc = do_pack(base, last)
code = code or nc
count += 1
else:
while len(outstanding) >= opt.jobs:
(pid,nc) = os.wait()
nc >>= 8
if pid in outstanding:
del outstanding[pid]
code = code or nc
count += 1
pid = os.fork()
if pid: # parent
outstanding[pid] = 1
else: # child
try:
sys.exit(do_pack(base, last))
except Exception, e:
log('exception: %r\n' % e)
sys.exit(99)
while len(outstanding):
(pid,nc) = os.wait()
nc >>= 8
if pid in outstanding:
del outstanding[pid]
code = code or nc
count += 1
if not opt.verbose:
progress('fsck (%d/%d)\r' % (count, len(extra)))
if not opt.verbose and istty:
log('fsck done. \n')
sys.exit(code)
|
