1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
|
"""
A simpler setup version just to compile the speedup module.
It should be used as:
python setup_pydevd_cython build_ext --inplace
Note: the .c file and other generated files are regenerated from
the .pyx file by running "python build_tools/build.py"
"""
import os
import sys
from setuptools import setup
os.chdir(os.path.dirname(os.path.abspath(__file__)))
IS_PY36_OR_GREATER = sys.version_info[:2] >= (3, 6)
IS_PY311_ONWARDS = sys.version_info[:2] >= (3, 11)
IS_PY312_ONWARDS = sys.version_info[:2] >= (3, 12)
def process_args():
extension_folder = None
target_pydevd_name = None
target_frame_eval = None
force_cython = False
for i, arg in enumerate(sys.argv[:]):
if arg == "--build-lib":
extension_folder = sys.argv[i + 1]
# It shouldn't be removed from sys.argv (among with --build-temp) because they're passed further to setup()
if arg.startswith("--target-pyd-name="):
sys.argv.remove(arg)
target_pydevd_name = arg[len("--target-pyd-name=") :]
if arg.startswith("--target-pyd-frame-eval="):
sys.argv.remove(arg)
target_frame_eval = arg[len("--target-pyd-frame-eval=") :]
if arg == "--force-cython":
sys.argv.remove(arg)
force_cython = True
return extension_folder, target_pydevd_name, target_frame_eval, force_cython
def process_template_lines(template_lines):
# Create 2 versions of the template, one for Python 3.8 and another for Python 3.9
for version in ("38", "39"):
yield "### WARNING: GENERATED CODE, DO NOT EDIT!"
yield "### WARNING: GENERATED CODE, DO NOT EDIT!"
yield "### WARNING: GENERATED CODE, DO NOT EDIT!"
for line in template_lines:
if version == "38":
line = line.replace(
"get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc)",
"get_bytecode_while_frame_eval_38(PyFrameObject * frame_obj, int exc)",
)
line = line.replace("CALL_EvalFrameDefault", "CALL_EvalFrameDefault_38(frame_obj, exc)")
else: # 3.9
line = line.replace(
"get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc)",
"get_bytecode_while_frame_eval_39(PyThreadState* tstate, PyFrameObject * frame_obj, int exc)",
)
line = line.replace("CALL_EvalFrameDefault", "CALL_EvalFrameDefault_39(tstate, frame_obj, exc)")
yield line
yield "### WARNING: GENERATED CODE, DO NOT EDIT!"
yield "### WARNING: GENERATED CODE, DO NOT EDIT!"
yield "### WARNING: GENERATED CODE, DO NOT EDIT!"
yield ""
yield ""
def process_template_file(contents):
ret = []
template_lines = []
append_to = ret
for line in contents.splitlines(keepends=False):
if line.strip() == "### TEMPLATE_START":
append_to = template_lines
elif line.strip() == "### TEMPLATE_END":
append_to = ret
for line in process_template_lines(template_lines):
ret.append(line)
else:
append_to.append(line)
return "\n".join(ret)
def build_extension(dir_name, extension_name, target_pydevd_name, force_cython, extended=False, has_pxd=False, template=False):
pyx_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.pyx" % (extension_name,))
if template:
pyx_template_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.template.pyx" % (extension_name,))
with open(pyx_template_file, "r") as stream:
contents = stream.read()
contents = process_template_file(contents)
with open(pyx_file, "w") as stream:
stream.write(contents)
if target_pydevd_name != extension_name:
# It MUST be there in this case!
# (otherwise we'll have unresolved externals because the .c file had another name initially).
import shutil
# We must force cython in this case (but only in this case -- for the regular setup in the user machine, we
# should always compile the .c file).
force_cython = True
new_pyx_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.pyx" % (target_pydevd_name,))
new_c_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.c" % (target_pydevd_name,))
shutil.copy(pyx_file, new_pyx_file)
pyx_file = new_pyx_file
if has_pxd:
pxd_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.pxd" % (extension_name,))
new_pxd_file = os.path.join(os.path.dirname(__file__), dir_name, "%s.pxd" % (target_pydevd_name,))
shutil.copy(pxd_file, new_pxd_file)
assert os.path.exists(pyx_file)
try:
c_files = [
os.path.join(dir_name, "%s.c" % target_pydevd_name),
]
if force_cython:
for c_file in c_files:
try:
os.remove(c_file)
except:
pass
from Cython.Build import cythonize # @UnusedImport
# Generate the .c files in cythonize (will not compile at this point).
target = "%s/%s.pyx" % (
dir_name,
target_pydevd_name,
)
cythonize([target])
# Workarounds needed in CPython 3.8 and 3.9 to access PyInterpreterState.eval_frame.
for c_file in c_files:
with open(c_file, "r") as stream:
c_file_contents = stream.read()
if '#include "internal/pycore_gc.h"' not in c_file_contents:
c_file_contents = c_file_contents.replace(
'#include "Python.h"',
"""#include "Python.h"
#if PY_VERSION_HEX >= 0x03090000
#include "internal/pycore_gc.h"
#include "internal/pycore_interp.h"
#endif
""",
)
if '#include "internal/pycore_pystate.h"' not in c_file_contents:
c_file_contents = c_file_contents.replace(
'#include "pystate.h"',
"""#include "pystate.h"
#if PY_VERSION_HEX >= 0x03080000
#include "internal/pycore_pystate.h"
#endif
""",
)
# We want the same output on Windows and Linux.
c_file_contents = c_file_contents.replace("\r\n", "\n").replace("\r", "\n")
c_file_contents = c_file_contents.replace(r"_pydevd_frame_eval\\release_mem.h", "_pydevd_frame_eval/release_mem.h")
c_file_contents = c_file_contents.replace(
r"_pydevd_frame_eval\\pydevd_frame_evaluator.pyx", "_pydevd_frame_eval/pydevd_frame_evaluator.pyx"
)
c_file_contents = c_file_contents.replace(r"_pydevd_bundle\\pydevd_cython.pxd", "_pydevd_bundle/pydevd_cython.pxd")
c_file_contents = c_file_contents.replace(r"_pydevd_bundle\\pydevd_cython.pyx", "_pydevd_bundle/pydevd_cython.pyx")
with open(c_file, "w") as stream:
stream.write(c_file_contents)
# Always compile the .c (and not the .pyx) file (which we should keep up-to-date by running build_tools/build.py).
from setuptools import Extension
extra_compile_args = []
extra_link_args = []
if "linux" in sys.platform:
# Enabling -flto brings executable from 4MB to 0.56MB and -Os to 0.41MB
# Profiling shows an execution around 3-5% slower with -Os vs -O3,
# so, kept only -flto.
extra_compile_args = ["-flto"]
extra_link_args = extra_compile_args[:]
# Note: also experimented with profile-guided optimization. The executable
# size became a bit smaller (from 0.56MB to 0.5MB) but this would add an
# extra step to run the debugger to obtain the optimizations
# so, skipped it for now (note: the actual benchmarks time was in the
# margin of a 0-1% improvement, which is probably not worth it for
# speed increments).
# extra_compile_args = ["-flto", "-fprofile-generate"]
# ... Run benchmarks ...
# extra_compile_args = ["-flto", "-fprofile-use", "-fprofile-correction"]
elif "win32" in sys.platform:
pass
# uncomment to generate pdbs for visual studio.
# extra_compile_args=["-Zi", "/Od"]
# extra_link_args=["-debug"]
if IS_PY311_ONWARDS:
# On py311 we need to add the CPython include folder to the include path.
extra_compile_args.append("-I%s\\include\\CPython" % sys.exec_prefix)
kwargs = {}
if extra_link_args:
kwargs["extra_link_args"] = extra_link_args
if extra_compile_args:
kwargs["extra_compile_args"] = extra_compile_args
ext_modules = [
Extension(
"%s%s.%s"
% (
dir_name,
"_ext" if extended else "",
target_pydevd_name,
),
c_files,
**kwargs,
)
]
# This is needed in CPython 3.8 to be able to include internal/pycore_pystate.h
# (needed to set PyInterpreterState.eval_frame).
for module in ext_modules:
module.define_macros = [("Py_BUILD_CORE_MODULE", "1")]
setup(name="Cythonize", ext_modules=ext_modules)
finally:
if target_pydevd_name != extension_name:
try:
os.remove(new_pyx_file)
except:
import traceback
traceback.print_exc()
try:
os.remove(new_c_file)
except:
import traceback
traceback.print_exc()
if has_pxd:
try:
os.remove(new_pxd_file)
except:
import traceback
traceback.print_exc()
extension_folder, target_pydevd_name, target_frame_eval, force_cython = process_args()
FORCE_BUILD_ALL = os.environ.get("PYDEVD_FORCE_BUILD_ALL", "").lower() in ("true", "1")
extension_name = "pydevd_cython"
if target_pydevd_name is None:
target_pydevd_name = extension_name
build_extension("_pydevd_bundle", extension_name, target_pydevd_name, force_cython, extension_folder, True)
if IS_PY36_OR_GREATER and not IS_PY311_ONWARDS or FORCE_BUILD_ALL:
extension_name = "pydevd_frame_evaluator"
if target_frame_eval is None:
target_frame_eval = extension_name
build_extension("_pydevd_frame_eval", extension_name, target_frame_eval, force_cython, extension_folder, True, template=True)
if IS_PY312_ONWARDS or FORCE_BUILD_ALL:
extension_name = "_pydevd_sys_monitoring_cython"
build_extension("_pydevd_sys_monitoring", extension_name, extension_name, force_cython, extension_folder, True)
if extension_folder:
os.chdir(extension_folder)
for folder in [
file for file in os.listdir(extension_folder) if file != "build" and os.path.isdir(os.path.join(extension_folder, file))
]:
file = os.path.join(folder, "__init__.py")
if not os.path.exists(file):
open(file, "a").close()
|
