1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
#!/usr/bin/python3
# encoding=utf-8
#
# Copyright © 2015 Alexandre Detiste <alexandre@detiste.be>
# SPDX-License-Identifier: GPL-2.0-or-later
# a small tool to create or verify a mirror
# of downloadable files needed by GDP
# call it this way:
# GDP_UNINSTALLED=1 python3 -m tools.mirror
# this tool will never remove any bad file,
# you'll have to investigate by yourself
import argparse
import os
import urllib.request
from game_data_packager.command_line import (TerminalProgress)
from game_data_packager.data import (HashedFile)
from game_data_packager.download import (Downloader)
from game_data_packager.game import (load_games)
from game_data_packager.util import (AGENT)
KEEP_FREE_SPACE = 250 * 1024 * 1024
class Archive:
filename: str
name: str
size: int
md5: str | None
sha1: str | None
sha256: str | None
download: str
os.environ.pop('GDP_MIRROR', None)
parser = argparse.ArgumentParser()
parser.add_argument('--destination', default='/var/www/html')
args = parser.parse_args()
print('loading game definitions...')
def load_archives() -> list[Archive]:
archives: list[Archive] = []
for gamename, game in load_games().items():
game.load_file_data()
for filename, file in game.files.items():
if file.unsuitable:
# quake2-rogue-2.00.tar.xz could have been tagged this way
fullname = os.path.join(args.destination, filename)
if os.path.isfile(fullname):
print('Obsolete archive: %s (%s)' %
(fullname, file.unsuitable))
continue
elif not file.download:
continue
url = Downloader.choose_mirror(file)[0]
if '?' not in url:
destname = os.path.basename(url)
elif '?' not in filename:
destname = filename
elif url.endswith('?download'):
destname = os.path.basename(url)
destname = destname[0:len(destname)-len('?download')]
else:
print("Can't compute filename for %s = %s" % (filename, url))
# bothersome hexen2demo_nov1997-linux-i586.tgz
continue
assert file.size, filename
assert file.md5 or file.sha1 or file.sha256, filename
archive = Archive()
archive.filename = filename
archive.name = destname
archive.size = file.size
archive.md5 = file.md5
archive.sha1 = file.sha1
archive.sha256 = file.sha256
archive.download = url
archives.append(archive)
return archives
archives = load_archives()
archives = sorted(archives, key=lambda k: (k.size))
seen: set[str] = set()
for a in archives:
# always prefer the smallest version of a file (e.g.: tnt31fix.zip)
if a.name in seen:
continue
seen.add(a.name)
archive = os.path.join(args.destination, a.name)
if not os.path.isfile(archive):
statvfs = os.statvfs(args.destination)
freespace = statvfs.f_frsize * statvfs.f_bavail
if a.size > freespace - KEEP_FREE_SPACE:
print('out of space, can not download %s' % a.name)
continue
try:
rf = urllib.request.urlopen(urllib.request.Request(a.download,
headers={'User-Agent': AGENT}))
except urllib.error.HTTPError as e:
print(e, a.name, a.download)
continue
if rf is None:
continue
with open(archive, 'wb') as wf:
hf = HashedFile.from_file(a.download, rf, wf, size=a.size,
progress=TerminalProgress(
info='downloading %s as %s' %
(a.download, a.name)))
rf.close()
else:
print('checking %s ...' % archive)
hf = HashedFile.from_file(archive,
open(archive, 'rb'),
size=a.size,
progress=TerminalProgress())
if os.path.getsize(archive) == 0:
exit("%s is empty !!!" % archive)
if os.path.getsize(archive) != a.size:
exit("%s has the wrong size !!!" % archive)
if a.md5 and a.md5 != hf.md5:
exit("md5 doesn't match for %s !!!" % archive)
if a.sha1 and a.sha1 != hf.sha1:
exit("sha1 doesn't match for %s !!!" % archive)
if a.sha256 and a.sha256 != hf.sha256:
exit("sha256 doesn't match for %s !!!" % archive)
|
