1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
#!/usr/bin/python3
# encoding=utf-8
#
# Copyright © 2015 Alexandre Detiste <alexandre@detiste.be>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# You can find the GPL license text on a Debian system under
# /usr/share/common-licenses/GPL-2.
# a small tool to create or verify a mirror
# of downloadable files needed by GDP
# call it this way:
# GDP_UNINSTALLED=1 python3 -m tools.mirror
# this tool will never remove any bad file,
# you'll have to investigate by yourself
KEEP_FREE_SPACE = 250 * 1024 * 1024
import argparse
import os
import urllib
from game_data_packager.command_line import (TerminalProgress)
from game_data_packager.data import (HashedFile)
from game_data_packager.download import (Downloader)
from game_data_packager.game import (load_games)
from game_data_packager.util import (AGENT)
archives = []
os.environ.pop('GDP_MIRROR', None)
parser = argparse.ArgumentParser()
parser.add_argument('--destination', default='/var/www/html')
args = parser.parse_args()
print('loading game definitions...')
for gamename, game in load_games().items():
game.load_file_data()
for filename, file in game.files.items():
if file.unsuitable:
# quake2-rogue-2.00.tar.xz could have been tagged this way
archive = os.path.join(args.destination, filename)
if os.path.isfile(archive):
print('Obsolete archive: %s (%s)' % (archive, file.unsuitable))
continue
elif not file.download:
continue
url = Downloader.choose_mirror(file)[0]
if '?' not in url:
destname = os.path.basename(url)
elif '?' not in filename:
destname = filename
elif url.endswith('?download'):
destname = os.path.basename(url)
destname = destname[0:len(destname)-len('?download')]
else:
print("Can't compute filename for %s = %s" % (filename, url))
# bothersome hexen2demo_nov1997-linux-i586.tgz
continue
archives.append({
'filename': filename,
'name': destname,
'size': file.size,
'md5': file.md5,
'sha1': file.sha1,
'download': url,
})
archives = sorted(archives, key=lambda k: (k['size']))
seen = set()
for a in archives:
# always prefer the smallest version of a file (e.g.: tnt31fix.zip)
if a['name'] in seen:
continue
seen.add(a['name'])
archive = os.path.join(args.destination, a['name'])
if not os.path.isfile(archive):
statvfs = os.statvfs(args.destination)
freespace = statvfs.f_frsize * statvfs.f_bavail
if a['size'] > freespace - KEEP_FREE_SPACE:
print('out of space, can not download %s' % a['name'])
continue
try:
rf = urllib.request.urlopen(urllib.request.Request(a['download'],
headers={'User-Agent': AGENT}))
except urllib.error.HTTPError as e:
print(e, a['name'], a['download'])
continue
if rf is None:
continue
wf = open(archive, 'wb')
hf = HashedFile.from_file(a['download'], rf, wf, size=a['size'],
progress=TerminalProgress(info='downloading %s as %s' %
(a['download'], a['name'])))
wf.close()
rf.close()
else:
print('checking %s ...' % archive)
hf = HashedFile.from_file(archive, open(archive, 'rb'),
size=a['size'], progress=TerminalProgress())
if os.path.getsize(archive) == 0:
exit("%s is empty !!!" % archive)
if os.path.getsize(archive) != a['size']:
exit("%s has the wrong size !!!" % archive)
if a['md5'] and a['md5'] != hf.md5:
exit("md5 doesn't match for %s !!!" % archive)
if a['sha1'] and a['sha1'] != hf.sha1:
exit("sha1 doesn't match for %s !!!" % archive)
|
