1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
|
"""
Highly experimental script that compiles the CPython standard library using Cython.
Execute the script either in the CPython 'Lib' directory or pass the
option '--current-python' to compile the standard library of the running
Python interpreter.
Pass '-j N' to get a parallel build with N processes.
Usage example::
$ python cystdlib.py --current-python build_ext -i
"""
import os
import sys
from distutils.core import setup
from Cython.Build import cythonize
from Cython.Compiler import Options
# improve Python compatibility by allowing some broken code
Options.error_on_unknown_names = False
Options.error_on_uninitialized = False
exclude_patterns = ['**/test/**/*.py', '**/tests/**/*.py', '**/__init__.py']
broken = [
'idlelib/MultiCall.py',
'email/utils.py',
'multiprocessing/reduction.py',
'multiprocessing/util.py',
'threading.py', # interrupt handling
'lib2to3/fixes/fix_sys_exc.py',
'traceback.py',
'types.py',
'enum.py',
'keyword.py',
'_collections_abc.py',
'importlib/_bootstrap',
]
default_directives = dict(
auto_cpdef=False, # enable when it's safe, see long list of failures below
binding=True,
set_initial_path='SOURCEFILE')
default_directives['optimize.inline_defnode_calls'] = True
special_directives = [
(['pkgutil.py',
'decimal.py',
'datetime.py',
'optparse.py',
'sndhdr.py',
'opcode.py',
'ntpath.py',
'urllib/request.py',
'plat-*/TYPES.py',
'plat-*/IN.py',
'tkinter/_fix.py',
'lib2to3/refactor.py',
'webbrowser.py',
'shutil.py',
'multiprocessing/forking.py',
'xml/sax/expatreader.py',
'xmlrpc/client.py',
'pydoc.py',
'xml/etree/ElementTree.py',
'posixpath.py',
'inspect.py',
'ctypes/util.py',
'urllib/parse.py',
'warnings.py',
'tempfile.py',
'trace.py',
'heapq.py',
'pickletools.py',
'multiprocessing/connection.py',
'hashlib.py',
'getopt.py',
'os.py',
'types.py',
], dict(auto_cpdef=False)),
]
del special_directives[:] # currently unused
def build_extensions(includes='**/*.py',
excludes=None,
special_directives=special_directives,
language_level=sys.version_info[0],
parallel=None):
if isinstance(includes, str):
includes = [includes]
excludes = list(excludes or exclude_patterns) + broken
all_groups = (special_directives or []) + [(includes, {})]
extensions = []
for modules, directives in all_groups:
exclude_now = excludes[:]
for other_modules, _ in special_directives:
if other_modules != modules:
exclude_now.extend(other_modules)
d = dict(default_directives)
d.update(directives)
extensions.extend(
cythonize(
modules,
exclude=exclude_now,
exclude_failures=True,
language_level=language_level,
compiler_directives=d,
nthreads=parallel,
))
return extensions
def build(extensions):
try:
setup(ext_modules=extensions)
result = True
except:
import traceback
print('error building extensions %s' % (
[ext.name for ext in extensions],))
traceback.print_exc()
result = False
return extensions, result
def _build(args):
sys_args, ext = args
sys.argv[1:] = sys_args
return build([ext])
def parse_args():
from optparse import OptionParser
parser = OptionParser('%prog [options] [LIB_DIR (default: ./Lib)]')
parser.add_option(
'--current-python', dest='current_python', action='store_true',
help='compile the stdlib of the running Python')
parser.add_option(
'-j', '--jobs', dest='parallel_jobs', metavar='N',
type=int, default=1,
help='run builds in N parallel jobs (default: 1)')
parser.add_option(
'-x', '--exclude', dest='excludes', metavar='PATTERN',
action="append", help='exclude modules/packages matching PATTERN')
options, args = parser.parse_args()
if not args:
args = ['./Lib']
elif len(args) > 1:
parser.error('only one argument expected, got %d' % len(args))
return options, args
if __name__ == '__main__':
options, args = parse_args()
if options.current_python:
# assume that the stdlib is where the "os" module lives
os.chdir(os.path.dirname(os.__file__))
else:
os.chdir(args[0])
pool = None
parallel_jobs = options.parallel_jobs
if options.parallel_jobs:
try:
import multiprocessing
pool = multiprocessing.Pool(parallel_jobs)
print("Building in %d parallel processes" % parallel_jobs)
except (ImportError, OSError):
print("Not building in parallel")
parallel_jobs = 0
extensions = build_extensions(
parallel=parallel_jobs,
excludes=options.excludes)
sys_args = ['build_ext', '-i']
if pool is not None:
results = pool.map(_build, [(sys_args, ext) for ext in extensions])
pool.close()
pool.join()
for ext, result in results:
if not result:
print("building extension %s failed" % (ext[0].name,))
else:
sys.argv[1:] = sys_args
build(extensions)
|
