1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
|
from rpython.jit.backend.ppc.arch import IS_PPC_64, WORD, PARAM_SAVE_AREA_OFFSET
from rpython.jit.backend.ppc.arch import THREADLOCAL_ADDR_OFFSET
import rpython.jit.backend.ppc.register as r
from rpython.jit.metainterp.history import INT, FLOAT
from rpython.jit.backend.llsupport.callbuilder import AbstractCallBuilder
from rpython.jit.backend.ppc.jump import remap_frame_layout
from rpython.rlib.objectmodel import we_are_translated
from rpython.jit.backend.llsupport import llerrno
from rpython.rtyper.lltypesystem import rffi
def follow_jump(addr):
# xxx implement me
return addr
class CallBuilder(AbstractCallBuilder):
GPR_ARGS = [r.r3, r.r4, r.r5, r.r6, r.r7, r.r8, r.r9, r.r10]
FPR_ARGS = r.MANAGED_FP_REGS
assert FPR_ARGS == [r.f1, r.f2, r.f3, r.f4, r.f5, r.f6, r.f7,
r.f8, r.f9, r.f10, r.f11, r.f12, r.f13]
RSHADOWPTR = r.RCS1
RFASTGILPTR = r.RCS2
RSHADOWOLD = r.RCS3
def __init__(self, assembler, fnloc, arglocs, resloc):
AbstractCallBuilder.__init__(self, assembler, fnloc, arglocs,
resloc, restype=INT, ressize=None)
def prepare_arguments(self):
assert IS_PPC_64
self.subtracted_to_sp = 0
# Prepare arguments. Note that this follows the convention where
# a prototype is in scope, and doesn't take "..." arguments. If
# you were to call a C function with a "..." argument with cffi,
# it would not go there but instead via libffi. If you pretend
# instead that it takes fixed arguments, then it would arrive here
# but the convention is bogus for floating-point arguments. (And,
# to add to the mess, at least CPython's ctypes cannot be used
# to call a "..." function with floating-point arguments. As I
# guess that it's a problem with libffi, it means PyPy inherits
# the same problem.)
arglocs = self.arglocs
num_args = len(arglocs)
non_float_locs = []
non_float_regs = []
float_locs = []
for i in range(min(num_args, 8)):
if arglocs[i].type != FLOAT:
non_float_locs.append(arglocs[i])
non_float_regs.append(self.GPR_ARGS[i])
else:
float_locs.append(arglocs[i])
# now 'non_float_locs' and 'float_locs' together contain the
# locations of the first 8 arguments
if num_args > 8:
# We need to make a larger PPC stack frame, as shown on the
# picture in arch.py. It needs to be 48 bytes + 8 * num_args.
# The new SP back chain location should point to the top of
# the whole stack frame, i.e. jumping over both the existing
# fixed-sise part and the new variable-sized part.
base = PARAM_SAVE_AREA_OFFSET
varsize = base + 8 * num_args
varsize = (varsize + 15) & ~15 # align
self.mc.load(r.SCRATCH2.value, r.SP.value, 0) # SP back chain
self.mc.store_update(r.SCRATCH2.value, r.SP.value, -varsize)
self.subtracted_to_sp = varsize
# In this variable-sized part, only the arguments from the 8th
# one need to be written, starting at SP + 112
for n in range(8, num_args):
loc = arglocs[n]
if loc.type != FLOAT:
# after the 8th argument, a non-float location is
# always stored in the stack
if loc.is_reg():
src = loc
else:
src = r.r2
self.asm.regalloc_mov(loc, src)
self.mc.std(src.value, r.SP.value, base + 8 * n)
else:
# the first 13 floating-point arguments are all passed
# in the registers f1 to f13, independently on their
# index in the complete list of arguments
if len(float_locs) < len(self.FPR_ARGS):
float_locs.append(loc)
else:
if loc.is_fp_reg():
src = loc
else:
src = r.FP_SCRATCH
self.asm.regalloc_mov(loc, src)
self.mc.stfd(src.value, r.SP.value, base + 8 * n)
# We must also copy fnloc into FNREG
non_float_locs.append(self.fnloc)
non_float_regs.append(self.mc.RAW_CALL_REG)
if float_locs:
assert len(float_locs) <= len(self.FPR_ARGS)
remap_frame_layout(self.asm, float_locs,
self.FPR_ARGS[:len(float_locs)],
r.FP_SCRATCH)
remap_frame_layout(self.asm, non_float_locs, non_float_regs,
r.SCRATCH)
def push_gcmap(self):
# we push *now* the gcmap, describing the status of GC registers
# after the rearrangements done just before, ignoring the return
# value r3, if necessary
assert not self.is_call_release_gil
noregs = self.asm.cpu.gc_ll_descr.is_shadow_stack()
gcmap = self.asm._regalloc.get_gcmap([r.r3], noregs=noregs)
self.asm.push_gcmap(self.mc, gcmap, store=True)
def pop_gcmap(self):
ssreg = None
gcrootmap = self.asm.cpu.gc_ll_descr.gcrootmap
if gcrootmap:
if gcrootmap.is_shadow_stack and self.is_call_release_gil:
# in this mode, RSHADOWOLD happens to contain the shadowstack
# top at this point, so reuse it instead of loading it again
ssreg = self.RSHADOWOLD
self.asm._reload_frame_if_necessary(self.mc, shadowstack_reg=ssreg)
def emit_raw_call(self):
self.mc.raw_call()
def restore_stack_pointer(self):
if self.subtracted_to_sp != 0:
self.mc.addi(r.SP.value, r.SP.value, self.subtracted_to_sp)
def load_result(self):
assert (self.resloc is None or
self.resloc is r.r3 or
self.resloc is r.f1)
def call_releasegil_addr_and_move_real_arguments(self, fastgil):
assert self.is_call_release_gil
RSHADOWPTR = self.RSHADOWPTR
RFASTGILPTR = self.RFASTGILPTR
RSHADOWOLD = self.RSHADOWOLD
#
# Save this thread's shadowstack pointer into r29, for later comparison
gcrootmap = self.asm.cpu.gc_ll_descr.gcrootmap
if gcrootmap:
if gcrootmap.is_shadow_stack:
rst = gcrootmap.get_root_stack_top_addr()
self.mc.load_imm(RSHADOWPTR, rst)
self.mc.load(RSHADOWOLD.value, RSHADOWPTR.value, 0)
#
# change 'rpy_fastgil' to 0 (it should be non-zero right now)
self.mc.load_imm(RFASTGILPTR, fastgil)
self.mc.li(r.r0.value, 0)
self.mc.lwsync()
self.mc.std(r.r0.value, RFASTGILPTR.value, 0)
#
if not we_are_translated(): # for testing: we should not access
self.mc.addi(r.SPP.value, r.SPP.value, 1) # r31 any more
def move_real_result_and_call_reacqgil_addr(self, fastgil):
from rpython.jit.backend.ppc.codebuilder import OverwritingBuilder
# try to reacquire the lock. The following registers are still
# valid from before the call:
RSHADOWPTR = self.RSHADOWPTR # r30: &root_stack_top
RFASTGILPTR = self.RFASTGILPTR # r29: &fastgil
RSHADOWOLD = self.RSHADOWOLD # r28: previous val of root_stack_top
# Equivalent of 'r10 = __sync_lock_test_and_set(&rpy_fastgil, 1);'
self.mc.li(r.r9.value, 1)
retry_label = self.mc.currpos()
self.mc.ldarx(r.r10.value, 0, RFASTGILPTR.value) # load the lock value
self.mc.stdcxx(r.r9.value, 0, RFASTGILPTR.value) # try to claim lock
self.mc.bc(6, 2, retry_label - self.mc.currpos()) # retry if failed
self.mc.isync()
self.mc.cmpdi(0, r.r10.value, 0)
b1_location = self.mc.currpos()
self.mc.trap() # boehm: patched with a BEQ: jump if r10 is zero
# shadowstack: patched with BNE instead
if self.asm.cpu.gc_ll_descr.gcrootmap:
# When doing a call_release_gil with shadowstack, there
# is the risk that the 'rpy_fastgil' was free but the
# current shadowstack can be the one of a different
# thread. So here we check if the shadowstack pointer
# is still the same as before we released the GIL (saved
# in RSHADOWOLD), and if not, we fall back to 'reacqgil_addr'.
self.mc.load(r.r9.value, RSHADOWPTR.value, 0)
self.mc.cmpdi(0, r.r9.value, RSHADOWOLD.value)
bne_location = b1_location
b1_location = self.mc.currpos()
self.mc.trap()
# revert the rpy_fastgil acquired above, so that the
# general 'reacqgil_addr' below can acquire it again...
# (here, r10 is conveniently zero)
self.mc.std(r.r10.value, RFASTGILPTR.value, 0)
pmc = OverwritingBuilder(self.mc, bne_location, 1)
pmc.bne(self.mc.currpos() - bne_location)
pmc.overwrite()
#
# Yes, we need to call the reacqgil() function.
# save the result we just got
RSAVEDRES = RFASTGILPTR # can reuse this reg here
reg = self.resloc
if reg is not None:
if reg.is_core_reg():
self.mc.mr(RSAVEDRES.value, reg.value)
elif reg.is_fp_reg():
self.mc.stfd(reg.value, r.SP.value,
PARAM_SAVE_AREA_OFFSET + 7 * WORD)
self.mc.load_imm(self.mc.RAW_CALL_REG, self.asm.reacqgil_addr)
self.mc.raw_call()
if reg is not None:
if reg.is_core_reg():
self.mc.mr(reg.value, RSAVEDRES.value)
elif reg.is_fp_reg():
self.mc.lfd(reg.value, r.SP.value,
PARAM_SAVE_AREA_OFFSET + 7 * WORD)
# replace b1_location with BEQ(here)
pmc = OverwritingBuilder(self.mc, b1_location, 1)
pmc.beq(self.mc.currpos() - b1_location)
pmc.overwrite()
if not we_are_translated(): # for testing: now we can access
self.mc.addi(r.SPP.value, r.SPP.value, -1) # r31 again
def write_real_errno(self, save_err):
if save_err & rffi.RFFI_READSAVED_ERRNO:
# Just before a call, read '*_errno' and write it into the
# real 'errno'. A lot of registers are free here, notably
# r11 and r0.
if save_err & rffi.RFFI_ALT_ERRNO:
rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu)
else:
rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
self.mc.ld(r.r11.value, r.SP.value,
THREADLOCAL_ADDR_OFFSET + self.subtracted_to_sp)
self.mc.lwz(r.r0.value, r.r11.value, rpy_errno)
self.mc.ld(r.r11.value, r.r11.value, p_errno)
self.mc.stw(r.r0.value, r.r11.value, 0)
elif save_err & rffi.RFFI_ZERO_ERRNO_BEFORE:
# Same, but write zero.
p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
self.mc.ld(r.r11.value, r.SP.value,
THREADLOCAL_ADDR_OFFSET + self.subtracted_to_sp)
self.mc.ld(r.r11.value, r.r11.value, p_errno)
self.mc.li(r.r0.value, 0)
self.mc.stw(r.r0.value, r.r11.value, 0)
def read_real_errno(self, save_err):
if save_err & rffi.RFFI_SAVE_ERRNO:
# Just after a call, read the real 'errno' and save a copy of
# it inside our thread-local '*_errno'. Registers r4-r10
# never contain anything after the call.
if save_err & rffi.RFFI_ALT_ERRNO:
rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu)
else:
rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
self.mc.ld(r.r9.value, r.SP.value, THREADLOCAL_ADDR_OFFSET)
self.mc.ld(r.r10.value, r.r9.value, p_errno)
self.mc.lwz(r.r10.value, r.r10.value, 0)
self.mc.stw(r.r10.value, r.r9.value, rpy_errno)
|
