from rpython.jit.backend.zarch.arch import WORD from rpython.jit.backend.zarch.arch import (THREADLOCAL_ADDR_OFFSET, STD_FRAME_SIZE_IN_BYTES) import rpython.jit.backend.zarch.locations as l import rpython.jit.backend.zarch.registers as r import rpython.jit.backend.zarch.conditions as c from rpython.jit.metainterp.history import INT, FLOAT from rpython.jit.backend.llsupport.callbuilder import AbstractCallBuilder from rpython.jit.backend.llsupport.jump import remap_frame_layout from rpython.rlib.objectmodel import we_are_translated from rpython.jit.backend.llsupport import llerrno from rpython.rtyper.lltypesystem import rffi from rpython.jit.backend.llsupport.descr import CallDescr CALL_RELEASE_GIL_STACK_OFF = 6*WORD class CallBuilder(AbstractCallBuilder): GPR_ARGS = [r.r2, r.r3, r.r4, r.r5, r.r6] FPR_ARGS = [r.f0, r.f2, r.f4, r.f6] RSHADOWOLD = r.r8 RSHADOWPTR = r.r9 RFASTGILPTR = r.r10 def __init__(self, assembler, fnloc, arglocs, resloc, calldescr): type = INT size = WORD self.ressign = True if calldescr is not None and isinstance(calldescr, CallDescr): type = calldescr.get_result_type() size = calldescr.get_result_size() self.ressign = calldescr.is_result_signed() AbstractCallBuilder.__init__(self, assembler, fnloc, arglocs, resloc, restype=type, ressize=size) def prepare_arguments(self): self.subtracted_to_sp = 0 # Prepare arguments. Note that this follows the convention where # a prototype is in scope, and doesn't take "..." arguments. If # you were to call a C function with a "..." argument with cffi, # it would not go there but instead via libffi. If you pretend # instead that it takes fixed arguments, then it would arrive here # but the convention is bogus for floating-point arguments. (And, # to add to the mess, at least CPython's ctypes cannot be used # to call a "..." function with floating-point arguments. As I # guess that it's a problem with libffi, it means PyPy inherits # the same problem.) arglocs = self.arglocs num_args = len(arglocs) max_gpr_in_reg = 5 max_fpr_in_reg = 4 non_float_locs = [] non_float_regs = [] float_locs = [] # the IBM zarch manual states: # """ # A function will be passed a frame on the runtime stack by the function which # called it, and may allocate a new stack frame. A new stack frame is required if the # called function will in turn call further functions (which must be passed the # address of the new frame). This stack grows downwards from high addresses # """ gpr_regs = 0 fpr_regs = 0 stack_params = [] for i in range(num_args): loc = arglocs[i] if not arglocs[i].is_float(): if gpr_regs < max_gpr_in_reg: non_float_locs.append(arglocs[i]) non_float_regs.append(self.GPR_ARGS[gpr_regs]) gpr_regs += 1 else: stack_params.append(i) else: if fpr_regs < max_fpr_in_reg: float_locs.append(arglocs[i]) fpr_regs += 1 else: stack_params.append(i) self.subtracted_to_sp += len(stack_params) * WORD base = len(stack_params) * WORD if self.is_call_release_gil: self.subtracted_to_sp += CALL_RELEASE_GIL_STACK_OFF base += CALL_RELEASE_GIL_STACK_OFF for idx,i in enumerate(stack_params): loc = arglocs[i] offset = STD_FRAME_SIZE_IN_BYTES - base + 8 * idx if loc.type == FLOAT: if loc.is_fp_reg(): src = loc else: src = r.FP_SCRATCH self.asm.regalloc_mov(loc, src) self.mc.STDY(src, l.addr(offset, r.SP)) else: if loc.is_core_reg(): src = loc else: src = r.SCRATCH self.asm.regalloc_mov(loc, src) self.mc.STG(src, l.addr(offset, r.SP)) # We must also copy fnloc into FNREG non_float_locs.append(self.fnloc) non_float_regs.append(r.RETURN) if float_locs: assert len(float_locs) <= len(self.FPR_ARGS) remap_frame_layout(self.asm, float_locs, self.FPR_ARGS[:len(float_locs)], r.FP_SCRATCH) remap_frame_layout(self.asm, non_float_locs, non_float_regs, r.SCRATCH) def push_gcmap(self): # we push *now* the gcmap, describing the status of GC registers # after the rearrangements done just before, ignoring the return # value r2, if necessary assert not self.is_call_release_gil noregs = self.asm.cpu.gc_ll_descr.is_shadow_stack() gcmap = self.asm._regalloc.get_gcmap([r.r2], noregs=noregs) self.asm.push_gcmap(self.mc, gcmap, store=True) def pop_gcmap(self): ssreg = None gcrootmap = self.asm.cpu.gc_ll_descr.gcrootmap if gcrootmap: if gcrootmap.is_shadow_stack and self.is_call_release_gil: # in this mode, RSHADOWOLD happens to contain the shadowstack # top at this point, so reuse it instead of loading it again # RSHADOWOLD is moved to the scratch reg just before restoring r8 ssreg = r.SCRATCH self.asm._reload_frame_if_necessary(self.mc, shadowstack_reg=ssreg) def emit_raw_call(self): # always allocate a stack frame for the new function # save the SP back chain # move the frame pointer if self.subtracted_to_sp != 0: # rewrite the back chain self.mc.LG(r.SCRATCH, l.addr(0, r.SP)) self.mc.STG(r.SCRATCH, l.addr(-self.subtracted_to_sp, r.SP)) self.mc.LAY(r.SP, l.addr(-self.subtracted_to_sp, r.SP)) self.mc.raw_call() def restore_stack_pointer(self): if self.subtracted_to_sp != 0: self.mc.LAY(r.SP, l.addr(self.subtracted_to_sp, r.SP)) def load_result(self): assert (self.resloc is None or self.resloc is r.GPR_RETURN or self.resloc is r.FPR_RETURN) # if self.restype == 'i' and self.ressize != WORD: # we must be sure! libffi (s390x impl) will not return # a sane 64 bit zero/sign extended value. fix for this # has been rejected (Jan. 2016). This will not be updated # any time soon... if self.ressign: # sign extend! if self.ressize == 1: self.mc.LGBR(r.r2, r.r2) elif self.ressize == 2: self.mc.LGHR(r.r2, r.r2) elif self.ressize == 4: self.mc.LGFR(r.r2, r.r2) else: assert 0, "cannot sign extend size %d" % self.ressize else: # zero extend! if self.ressize == 1: self.mc.LLGCR(r.r2, r.r2) elif self.ressize == 2: self.mc.LLGHR(r.r2, r.r2) elif self.ressize == 4: self.mc.LLGFR(r.r2, r.r2) else: assert 0, "cannot zero extend size %d" % self.ressize def call_releasegil_addr_and_move_real_arguments(self, fastgil): assert self.is_call_release_gil RSHADOWOLD = self.RSHADOWOLD RSHADOWPTR = self.RSHADOWPTR RFASTGILPTR = self.RFASTGILPTR # pos = STD_FRAME_SIZE_IN_BYTES - CALL_RELEASE_GIL_STACK_OFF self.mc.STMG(r.r8, r.r13, l.addr(pos, r.SP)) # # Save this thread's shadowstack pointer into r8, for later comparison gcrootmap = self.asm.cpu.gc_ll_descr.gcrootmap if gcrootmap: if gcrootmap.is_shadow_stack: rst = gcrootmap.get_root_stack_top_addr() self.mc.load_imm(RSHADOWPTR, rst) self.mc.load(RSHADOWOLD, RSHADOWPTR, 0) # # change 'rpy_fastgil' to 0 (it should be non-zero right now) self.mc.load_imm(RFASTGILPTR, fastgil) self.mc.XGR(r.SCRATCH, r.SCRATCH) # zarch is sequentially consistent self.mc.STG(r.SCRATCH, l.addr(0, RFASTGILPTR)) def move_real_result_and_call_reacqgil_addr(self, fastgil): from rpython.jit.backend.zarch.codebuilder import OverwritingBuilder # try to reacquire the lock. The following registers are still # valid from before the call: RSHADOWOLD = self.RSHADOWOLD # r8: previous val of root_stack_top RSHADOWPTR = self.RSHADOWPTR # r9: &root_stack_top RFASTGILPTR = self.RFASTGILPTR # r10: &fastgil # Equivalent of 'r13 = __sync_lock_test_and_set(&rpy_fastgil, 1);' self.mc.LGHI(r.SCRATCH, l.imm(1)) self.mc.LG(r.r13, l.addr(0, RFASTGILPTR)) retry_label = self.mc.currpos() self.mc.LGR(r.r14, r.r13) self.mc.CSG(r.r13, r.SCRATCH, l.addr(0, RFASTGILPTR)) # try to claim lock self.mc.BRC(c.LT, l.imm(retry_label - self.mc.currpos())) # retry if failed # CSG performs a serialization # zarch is sequential consistent! self.mc.CGHI(r.r14, l.imm0) b1_location = self.mc.currpos() # boehm: patched with a BEQ: jump if r13 is zero # shadowstack: patched with BNE instead self.mc.reserve_cond_jump() gcrootmap = self.asm.cpu.gc_ll_descr.gcrootmap if gcrootmap: # When doing a call_release_gil with shadowstack, there # is the risk that the 'rpy_fastgil' was free but the # current shadowstack can be the one of a different # thread. So here we check if the shadowstack pointer # is still the same as before we released the GIL (saved # in RSHADOWOLD), and if not, we fall back to 'reacqgil_addr'. self.mc.load(r.SCRATCH, RSHADOWPTR, 0) self.mc.CGR(r.SCRATCH, RSHADOWOLD) bne_location = b1_location b1_location = self.mc.currpos() self.mc.reserve_cond_jump() # revert the rpy_fastgil acquired above, so that the # general 'reacqgil_addr' below can acquire it again... # (here, r14 is conveniently zero) self.mc.STG(r.r14, l.addr(0, RFASTGILPTR)) pmc = OverwritingBuilder(self.mc, bne_location, 1) pmc.BRCL(c.NE, l.imm(self.mc.currpos() - bne_location)) pmc.overwrite() # # Yes, we need to call the reacqgil() function. # save the result we just got RSAVEDRES = RFASTGILPTR # can reuse this reg here reg = self.resloc PARAM_SAVE_AREA_OFFSET = 0 if reg is not None: # save 1 word below the stack pointer if reg.is_core_reg(): self.mc.LGR(RSAVEDRES, reg) elif reg.is_fp_reg(): self.mc.STD(reg, l.addr(16*WORD, r.SP)) # r8-r13 live on the stack and must NOT be overwritten, # restore_stack_pointer already moved SP + subtracted_to_sp, self.mc.LAY(r.SP, l.addr(-self.subtracted_to_sp, r.SP)) self.mc.load_imm(self.mc.RAW_CALL_REG, self.asm.reacqgil_addr) self.mc.raw_call() self.mc.LAY(r.SP, l.addr(self.subtracted_to_sp, r.SP)) if reg is not None: if reg.is_core_reg(): self.mc.LGR(reg, RSAVEDRES) elif reg.is_fp_reg(): self.mc.LD(reg, l.addr(16*WORD, r.SP)) # replace b1_location with BEQ(here) pmc = OverwritingBuilder(self.mc, b1_location, 1) pmc.BRCL(c.EQ, l.imm(self.mc.currpos() - b1_location)) pmc.overwrite() if gcrootmap: if gcrootmap.is_shadow_stack and self.is_call_release_gil: self.mc.LGR(r.SCRATCH, RSHADOWOLD) pos = STD_FRAME_SIZE_IN_BYTES - CALL_RELEASE_GIL_STACK_OFF self.mc.LMG(r.r8, r.r13, l.addr(pos, r.SP)) def write_real_errno(self, save_err): # r11 is saved in call_releasegil_addr_and_move_real_arguments, # thus can be used freely here! if save_err & rffi.RFFI_READSAVED_ERRNO: # Just before a call, read '*_errno' and write it into the # real 'errno'. if save_err & rffi.RFFI_ALT_ERRNO: rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu) else: rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu) p_errno = llerrno.get_p_errno_offset(self.asm.cpu) self.mc.LG(r.r11, l.addr(THREADLOCAL_ADDR_OFFSET, r.SP)) self.mc.LGF(r.SCRATCH2, l.addr(rpy_errno, r.r11)) self.mc.LG(r.r11, l.addr(p_errno, r.r11)) self.mc.STY(r.SCRATCH2, l.addr(0,r.r11)) elif save_err & rffi.RFFI_ZERO_ERRNO_BEFORE: # Same, but write zero. p_errno = llerrno.get_p_errno_offset(self.asm.cpu) self.mc.LG(r.r11, l.addr(THREADLOCAL_ADDR_OFFSET, r.SP)) self.mc.LG(r.r11, l.addr(p_errno, r.r11)) self.mc.LGHI(r.SCRATCH, l.imm(0)) self.mc.STY(r.SCRATCH, l.addr(0,r.r11)) def read_real_errno(self, save_err): if save_err & rffi.RFFI_SAVE_ERRNO: # Just after a call, read the real 'errno' and save a copy of # it inside our thread-local '*_errno'. Registers r3-r6 # never contain anything after the call. if save_err & rffi.RFFI_ALT_ERRNO: rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu) else: rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu) p_errno = llerrno.get_p_errno_offset(self.asm.cpu) self.mc.LG(r.r3, l.addr(THREADLOCAL_ADDR_OFFSET, r.SP)) self.mc.LG(r.r4, l.addr(p_errno, r.r3)) self.mc.LGF(r.r4, l.addr(0, r.r4)) self.mc.STY(r.r4, l.addr(rpy_errno, r.r3))