""" x86 Encoding recipes. """ from __future__ import absolute_import from cdsl.isa import EncRecipe from cdsl.predicates import IsSignedInt, IsEqual, Or from cdsl.predicates import IsZero32BitFloat, IsZero64BitFloat from cdsl.registers import RegClass from base.formats import Unary, UnaryIeee32, UnaryIeee64, UnaryImm, UnaryBool from base.formats import Binary, BinaryImm from base.formats import MultiAry, NullAry from base.formats import Trap, Call, CallIndirect, Store, Load from base.formats import IntCompare, IntCompareImm, FloatCompare from base.formats import IntCond, FloatCond from base.formats import IntSelect, IntCondTrap, FloatCondTrap from base.formats import Jump, Branch, BranchInt, BranchFloat from base.formats import BranchTableEntry, BranchTableBase, IndirectJump from base.formats import Ternary, FuncAddr, UnaryGlobalValue from base.formats import RegMove, RegSpill, RegFill, CopySpecial from base.formats import LoadComplex, StoreComplex from base.formats import StackLoad from .registers import GPR, ABCD, FPR from .registers import GPR8, FPR8, FLAG from .registers import StackGPR32, StackFPR32 from .defs import supported_floatccs from .settings import use_sse41 try: from typing import Tuple, Dict, Sequence, Any # noqa from cdsl.instructions import InstructionFormat # noqa from cdsl.isa import ConstraintSeq, BranchRange, PredNode, OperandConstraint # noqa except ImportError: pass # Opcode representation. # # Cranelift requires each recipe to have a single encoding size in bytes, and # x86 opcodes are variable length, so we use separate recipes for different # styles of opcodes and prefixes. The opcode format is indicated by the recipe # name prefix: OPCODE_PREFIX = { # Prefix bytes Name mmpp (): ('Op1', 0b0000), (0x66,): ('Mp1', 0b0001), (0xf3,): ('Mp1', 0b0010), (0xf2,): ('Mp1', 0b0011), (0x0f,): ('Op2', 0b0100), (0x66, 0x0f): ('Mp2', 0b0101), (0xf3, 0x0f): ('Mp2', 0b0110), (0xf2, 0x0f): ('Mp2', 0b0111), (0x0f, 0x38): ('Op3', 0b1000), (0x66, 0x0f, 0x38): ('Mp3', 0b1001), (0xf3, 0x0f, 0x38): ('Mp3', 0b1010), (0xf2, 0x0f, 0x38): ('Mp3', 0b1011), (0x0f, 0x3a): ('Op3', 0b1100), (0x66, 0x0f, 0x3a): ('Mp3', 0b1101), (0xf3, 0x0f, 0x3a): ('Mp3', 0b1110), (0xf2, 0x0f, 0x3a): ('Mp3', 0b1111) } # The table above does not include the REX prefix which goes after the # mandatory prefix. VEX/XOP and EVEX prefixes are not yet supported. Encodings # using any of these prefixes are represented by separate recipes. # # The encoding bits are: # # 0-7: The opcode byte . # 8-9: pp, mandatory prefix: # 00 none (Op*) # 01 66 (Mp*) # 10 F3 (Mp*) # 11 F2 (Mp*) # 10-11: mm, opcode map: # 00 (Op1/Mp1) # 01 0F (Op2/Mp2) # 10 0F 38 (Op3/Mp3) # 11 0F 3A (Op3/Mp3) # 12-14 rrr, opcode bits for the ModR/M byte for certain opcodes. # 15: REX.W bit (or VEX.W/E) # # There is some redundancy between bits 8-11 and the recipe names, but we have # enough bits, and the pp+mm format is ready for supporting VEX prefixes. def decode_ops(ops, rrr=0, w=0): # type: (Tuple[int, ...], int, int) -> Tuple[str, int] """ Given a sequence of opcode bytes, compute the recipe name prefix and encoding bits. """ assert rrr <= 0b111 assert w <= 1 name, mmpp = OPCODE_PREFIX[ops[:-1]] op = ops[-1] assert op <= 256 return (name, op | (mmpp << 8) | (rrr << 12) | (w << 15)) def replace_put_op(emit, prefix): # type: (str, str) -> str """ Given a snippet of Rust code (or None), replace the `PUT_OP` macro with the corresponding `put_*` function from the `binemit.rs` module. """ if emit is None: return None else: return emit.replace('PUT_OP', 'put_' + prefix.lower()) # Register class mapping for no-REX instructions. NOREX_MAP = { GPR: GPR8, FPR: FPR8 } def map_regs_norex(regs): # type: (Sequence[OperandConstraint]) -> Sequence[OperandConstraint] return tuple(NOREX_MAP.get(rc, rc) if isinstance(rc, RegClass) else rc for rc in regs) class TailRecipe: """ Generate encoding recipes on demand. x86 encodings are somewhat orthogonal with the opcode representation on one side and the ModR/M, SIB and immediate fields on the other side. A `TailRecipe` represents the part of an encoding that follow the opcode. It is used to generate full encoding recipes on demand when combined with an opcode. The arguments are the same as for an `EncRecipe`, except for `size` which does not include the size of the opcode. The `when_prefixed` parameter specifies a recipe that should be substituted for this one when a REX (or VEX) prefix is present. This is relevant for recipes that can only access the ABCD registers without a REX prefix, but are able to access all registers with a prefix. The `requires_prefix` parameter indicates that the recipe can't be used without a REX prefix. The `emit` parameter contains Rust code to actually emit an encoding, like `EncRecipe` does it. Additionally, the text `PUT_OP` is substituted with the proper `put_*` function from the `x86/binemit.rs` module. """ def __init__( self, name, # type: str format, # type: InstructionFormat base_size, # type: int ins, # type: ConstraintSeq outs, # type: ConstraintSeq branch_range=None, # type: int clobbers_flags=True, # type: bool instp=None, # type: PredNode isap=None, # type: PredNode when_prefixed=None, # type: TailRecipe requires_prefix=False, # type: bool emit=None, # type: str compute_size=None # type: str ): # type: (...) -> None self.name = name self.format = format self.base_size = base_size self.ins = ins self.outs = outs self.branch_range = branch_range self.clobbers_flags = clobbers_flags self.instp = instp self.isap = isap self.when_prefixed = when_prefixed self.requires_prefix = requires_prefix self.emit = emit self.compute_size = compute_size # Cached recipes, keyed by name prefix. self.recipes = dict() # type: Dict[str, EncRecipe] def __call__(self, *ops, **kwargs): # type: (*int, **int) -> Tuple[EncRecipe, int] """ Create an encoding recipe and encoding bits for the opcode bytes in `ops`. """ assert not self.requires_prefix, "Tail recipe requires REX prefix." rrr = kwargs.get('rrr', 0) w = kwargs.get('w', 0) name, bits = decode_ops(ops, rrr, w) base_size = len(ops) + self.base_size # All branch ranges are relative to the end of the instruction. branch_range = None # type BranchRange if self.branch_range is not None: branch_range = (base_size, self.branch_range) if name not in self.recipes: recipe = EncRecipe( name + self.name, self.format, base_size, ins=self.ins, outs=self.outs, branch_range=branch_range, clobbers_flags=self.clobbers_flags, instp=self.instp, isap=self.isap, emit=replace_put_op(self.emit, name), compute_size=self.compute_size) recipe.ins = map_regs_norex(recipe.ins) recipe.outs = map_regs_norex(recipe.outs) self.recipes[name] = recipe return (self.recipes[name], bits) def rex(self, *ops, **kwargs): # type: (*int, **int) -> Tuple[EncRecipe, int] """ Create a REX encoding recipe and encoding bits for the opcode bytes in `ops`. The recipe will always generate a REX prefix, whether it is required or not. For instructions that don't require a REX prefix, two encodings should be added: One with REX and one without. """ # Use the prefixed alternative recipe when applicable. if self.when_prefixed: return self.when_prefixed.rex(*ops, **kwargs) rrr = kwargs.get('rrr', 0) w = kwargs.get('w', 0) name, bits = decode_ops(ops, rrr, w) name = 'Rex' + name base_size = 1 + len(ops) + self.base_size # All branch ranges are relative to the end of the instruction. branch_range = None # type BranchRange if self.branch_range is not None: branch_range = (base_size, self.branch_range) if name not in self.recipes: recipe = EncRecipe( name + self.name, self.format, base_size, ins=self.ins, outs=self.outs, branch_range=branch_range, clobbers_flags=self.clobbers_flags, instp=self.instp, isap=self.isap, emit=replace_put_op(self.emit, name), compute_size=self.compute_size) self.recipes[name] = recipe return (self.recipes[name], bits) @staticmethod def check_names(globs): # type: (Dict[str, Any]) -> None for name, obj in globs.items(): if isinstance(obj, TailRecipe): assert name == obj.name, "Mismatched TailRecipe name: " + name def floatccs(iform): # type: (InstructionFormat) -> PredNode """ Return an instruction predicate that checks in `iform.cond` is one of the directly supported floating point condition codes. """ return Or(*(IsEqual(iform.cond, cc) for cc in supported_floatccs)) def valid_scale(iform): # type: (InstructionFormat) -> PredNode """ Return an instruction predicate that checks if `iform.imm` is a valid `scale` for a SIB byte. """ return Or(IsEqual(iform.imm, 1), IsEqual(iform.imm, 2), IsEqual(iform.imm, 4), IsEqual(iform.imm, 8)) # A null unary instruction that takes a GPR register. Can be used for identity # copies and no-op conversions. null = EncRecipe('null', Unary, base_size=0, ins=GPR, outs=0, emit='') debugtrap = EncRecipe('debugtrap', NullAry, base_size=1, ins=(), outs=(), emit=''' sink.put1(0xcc); ''') # XX opcode, no ModR/M. trap = TailRecipe( 'trap', Trap, base_size=0, ins=(), outs=(), emit=''' sink.trap(code, func.srclocs[inst]); PUT_OP(bits, BASE_REX, sink); ''') # Macro: conditional jump over a ud2. trapif = EncRecipe( 'trapif', IntCondTrap, base_size=4, ins=FLAG.rflags, outs=(), clobbers_flags=False, emit=''' // Jump over a 2-byte ud2. sink.put1(0x70 | (icc2opc(cond.inverse()) as u8)); sink.put1(2); // ud2. sink.trap(code, func.srclocs[inst]); sink.put1(0x0f); sink.put1(0x0b); ''') trapff = EncRecipe( 'trapff', FloatCondTrap, base_size=4, ins=FLAG.rflags, outs=(), clobbers_flags=False, instp=floatccs(FloatCondTrap), emit=''' // Jump over a 2-byte ud2. sink.put1(0x70 | (fcc2opc(cond.inverse()) as u8)); sink.put1(2); // ud2. sink.trap(code, func.srclocs[inst]); sink.put1(0x0f); sink.put1(0x0b); ''') # XX /r rr = TailRecipe( 'rr', Binary, base_size=1, ins=(GPR, GPR), outs=0, emit=''' PUT_OP(bits, rex2(in_reg0, in_reg1), sink); modrm_rr(in_reg0, in_reg1, sink); ''') # XX /r with operands swapped. (RM form). rrx = TailRecipe( 'rrx', Binary, base_size=1, ins=(GPR, GPR), outs=0, emit=''' PUT_OP(bits, rex2(in_reg1, in_reg0), sink); modrm_rr(in_reg1, in_reg0, sink); ''') # XX /r with FPR ins and outs. A form. fa = TailRecipe( 'fa', Binary, base_size=1, ins=(FPR, FPR), outs=0, emit=''' PUT_OP(bits, rex2(in_reg1, in_reg0), sink); modrm_rr(in_reg1, in_reg0, sink); ''') # XX /r with FPR ins and outs. A form with input operands swapped. fax = TailRecipe( 'fax', Binary, base_size=1, ins=(FPR, FPR), outs=1, emit=''' PUT_OP(bits, rex2(in_reg0, in_reg1), sink); modrm_rr(in_reg0, in_reg1, sink); ''') # XX /n for a unary operation with extension bits. ur = TailRecipe( 'ur', Unary, base_size=1, ins=GPR, outs=0, emit=''' PUT_OP(bits, rex1(in_reg0), sink); modrm_r_bits(in_reg0, bits, sink); ''') # XX /r, but for a unary operator with separate input/output register, like # copies. MR form, preserving flags. umr = TailRecipe( 'umr', Unary, base_size=1, ins=GPR, outs=GPR, clobbers_flags=False, emit=''' PUT_OP(bits, rex2(out_reg0, in_reg0), sink); modrm_rr(out_reg0, in_reg0, sink); ''') # Same as umr, but with FPR -> GPR registers. rfumr = TailRecipe( 'rfumr', Unary, base_size=1, ins=FPR, outs=GPR, clobbers_flags=False, emit=''' PUT_OP(bits, rex2(out_reg0, in_reg0), sink); modrm_rr(out_reg0, in_reg0, sink); ''') # XX /r, but for a unary operator with separate input/output register. # RM form. Clobbers FLAGS. urm = TailRecipe( 'urm', Unary, base_size=1, ins=GPR, outs=GPR, emit=''' PUT_OP(bits, rex2(in_reg0, out_reg0), sink); modrm_rr(in_reg0, out_reg0, sink); ''') # XX /r. Same as urm, but doesn't clobber FLAGS. urm_noflags = TailRecipe( 'urm_noflags', Unary, base_size=1, ins=GPR, outs=GPR, clobbers_flags=False, emit=''' PUT_OP(bits, rex2(in_reg0, out_reg0), sink); modrm_rr(in_reg0, out_reg0, sink); ''') # XX /r. Same as urm_noflags, but input limited to ABCD. urm_noflags_abcd = TailRecipe( 'urm_noflags_abcd', Unary, base_size=1, ins=ABCD, outs=GPR, when_prefixed=urm_noflags, clobbers_flags=False, emit=''' PUT_OP(bits, rex2(in_reg0, out_reg0), sink); modrm_rr(in_reg0, out_reg0, sink); ''') # XX /r, RM form, FPR -> FPR. furm = TailRecipe( 'furm', Unary, base_size=1, ins=FPR, outs=FPR, clobbers_flags=False, emit=''' PUT_OP(bits, rex2(in_reg0, out_reg0), sink); modrm_rr(in_reg0, out_reg0, sink); ''') # XX /r, RM form, GPR -> FPR. frurm = TailRecipe( 'frurm', Unary, base_size=1, ins=GPR, outs=FPR, clobbers_flags=False, emit=''' PUT_OP(bits, rex2(in_reg0, out_reg0), sink); modrm_rr(in_reg0, out_reg0, sink); ''') # XX /r, RM form, FPR -> GPR. rfurm = TailRecipe( 'rfurm', Unary, base_size=1, ins=FPR, outs=GPR, clobbers_flags=False, emit=''' PUT_OP(bits, rex2(in_reg0, out_reg0), sink); modrm_rr(in_reg0, out_reg0, sink); ''') # XX /r, RMI form for one of the roundXX SSE 4.1 instructions. furmi_rnd = TailRecipe( 'furmi_rnd', Unary, base_size=2, ins=FPR, outs=FPR, isap=use_sse41, emit=''' PUT_OP(bits, rex2(in_reg0, out_reg0), sink); modrm_rr(in_reg0, out_reg0, sink); sink.put1(match opcode { Opcode::Nearest => 0b00, Opcode::Floor => 0b01, Opcode::Ceil => 0b10, Opcode::Trunc => 0b11, x => panic!("{} unexpected for furmi_rnd", opcode), }); ''') # XX /r, for regmove instructions. rmov = TailRecipe( 'rmov', RegMove, base_size=1, ins=GPR, outs=(), clobbers_flags=False, emit=''' PUT_OP(bits, rex2(dst, src), sink); modrm_rr(dst, src, sink); ''') # XX /r, for regmove instructions (FPR version, RM encoded). frmov = TailRecipe( 'frmov', RegMove, base_size=1, ins=FPR, outs=(), clobbers_flags=False, emit=''' PUT_OP(bits, rex2(src, dst), sink); modrm_rr(src, dst, sink); ''') # XX /n with one arg in %rcx, for shifts. rc = TailRecipe( 'rc', Binary, base_size=1, ins=(GPR, GPR.rcx), outs=0, emit=''' PUT_OP(bits, rex1(in_reg0), sink); modrm_r_bits(in_reg0, bits, sink); ''') # XX /n for division: inputs in %rax, %rdx, r. Outputs in %rax, %rdx. div = TailRecipe( 'div', Ternary, base_size=1, ins=(GPR.rax, GPR.rdx, GPR), outs=(GPR.rax, GPR.rdx), emit=''' sink.trap(TrapCode::IntegerDivisionByZero, func.srclocs[inst]); PUT_OP(bits, rex1(in_reg2), sink); modrm_r_bits(in_reg2, bits, sink); ''') # XX /n for {s,u}mulx: inputs in %rax, r. Outputs in %rdx(hi):%rax(lo) mulx = TailRecipe( 'mulx', Binary, base_size=1, ins=(GPR.rax, GPR), outs=(GPR.rax, GPR.rdx), emit=''' PUT_OP(bits, rex1(in_reg1), sink); modrm_r_bits(in_reg1, bits, sink); ''') # XX /n ib with 8-bit immediate sign-extended. r_ib = TailRecipe( 'r_ib', BinaryImm, base_size=2, ins=GPR, outs=0, instp=IsSignedInt(BinaryImm.imm, 8), emit=''' PUT_OP(bits, rex1(in_reg0), sink); modrm_r_bits(in_reg0, bits, sink); let imm: i64 = imm.into(); sink.put1(imm as u8); ''') # XX /n id with 32-bit immediate sign-extended. r_id = TailRecipe( 'r_id', BinaryImm, base_size=5, ins=GPR, outs=0, instp=IsSignedInt(BinaryImm.imm, 32), emit=''' PUT_OP(bits, rex1(in_reg0), sink); modrm_r_bits(in_reg0, bits, sink); let imm: i64 = imm.into(); sink.put4(imm as u32); ''') # XX /n id with 32-bit immediate sign-extended. UnaryImm version. u_id = TailRecipe( 'u_id', UnaryImm, base_size=5, ins=(), outs=GPR, instp=IsSignedInt(UnaryImm.imm, 32), emit=''' PUT_OP(bits, rex1(out_reg0), sink); modrm_r_bits(out_reg0, bits, sink); let imm: i64 = imm.into(); sink.put4(imm as u32); ''') # XX+rd id unary with 32-bit immediate. Note no recipe predicate. pu_id = TailRecipe( 'pu_id', UnaryImm, base_size=4, ins=(), outs=GPR, emit=''' // The destination register is encoded in the low bits of the opcode. // No ModR/M. PUT_OP(bits | (out_reg0 & 7), rex1(out_reg0), sink); let imm: i64 = imm.into(); sink.put4(imm as u32); ''') # XX+rd id unary with bool immediate. Note no recipe predicate. pu_id_bool = TailRecipe( 'pu_id_bool', UnaryBool, base_size=4, ins=(), outs=GPR, emit=''' // The destination register is encoded in the low bits of the opcode. // No ModR/M. PUT_OP(bits | (out_reg0 & 7), rex1(out_reg0), sink); let imm: u32 = if imm { 1 } else { 0 }; sink.put4(imm); ''') # XX+rd iq unary with 64-bit immediate. pu_iq = TailRecipe( 'pu_iq', UnaryImm, base_size=8, ins=(), outs=GPR, emit=''' PUT_OP(bits | (out_reg0 & 7), rex1(out_reg0), sink); let imm: i64 = imm.into(); sink.put8(imm as u64); ''') # XX /n Unary with floating point 32-bit immediate equal to zero. f32imm_z = TailRecipe( 'f32imm_z', UnaryIeee32, base_size=1, ins=(), outs=FPR, instp=IsZero32BitFloat(UnaryIeee32.imm), emit=''' PUT_OP(bits, rex2(out_reg0, out_reg0), sink); modrm_rr(out_reg0, out_reg0, sink); ''') # XX /n Unary with floating point 64-bit immediate equal to zero. f64imm_z = TailRecipe( 'f64imm_z', UnaryIeee64, base_size=1, ins=(), outs=FPR, instp=IsZero64BitFloat(UnaryIeee64.imm), emit=''' PUT_OP(bits, rex2(out_reg0, out_reg0), sink); modrm_rr(out_reg0, out_reg0, sink); ''') pushq = TailRecipe( 'pushq', Unary, base_size=0, ins=GPR, outs=(), emit=''' sink.trap(TrapCode::StackOverflow, func.srclocs[inst]); PUT_OP(bits | (in_reg0 & 7), rex1(in_reg0), sink); ''') popq = TailRecipe( 'popq', NullAry, base_size=0, ins=(), outs=GPR, emit=''' PUT_OP(bits | (out_reg0 & 7), rex1(out_reg0), sink); ''') # XX /r, for regmove instructions. copysp = TailRecipe( 'copysp', CopySpecial, base_size=1, ins=(), outs=(), clobbers_flags=False, emit=''' PUT_OP(bits, rex2(dst, src), sink); modrm_rr(dst, src, sink); ''') adjustsp = TailRecipe( 'adjustsp', Unary, base_size=1, ins=(GPR), outs=(), emit=''' PUT_OP(bits, rex2(RU::rsp.into(), in_reg0), sink); modrm_rr(RU::rsp.into(), in_reg0, sink); ''') adjustsp_ib = TailRecipe( 'adjustsp_ib', UnaryImm, base_size=2, ins=(), outs=(), instp=IsSignedInt(UnaryImm.imm, 8), emit=''' PUT_OP(bits, rex1(RU::rsp.into()), sink); modrm_r_bits(RU::rsp.into(), bits, sink); let imm: i64 = imm.into(); sink.put1(imm as u8); ''') adjustsp_id = TailRecipe( 'adjustsp_id', UnaryImm, base_size=5, ins=(), outs=(), instp=IsSignedInt(UnaryImm.imm, 32), emit=''' PUT_OP(bits, rex1(RU::rsp.into()), sink); modrm_r_bits(RU::rsp.into(), bits, sink); let imm: i64 = imm.into(); sink.put4(imm as u32); ''') # XX+rd id with Abs4 function relocation. fnaddr4 = TailRecipe( 'fnaddr4', FuncAddr, base_size=4, ins=(), outs=GPR, emit=''' PUT_OP(bits | (out_reg0 & 7), rex1(out_reg0), sink); sink.reloc_external(Reloc::Abs4, &func.dfg.ext_funcs[func_ref].name, 0); sink.put4(0); ''') # XX+rd iq with Abs8 function relocation. fnaddr8 = TailRecipe( 'fnaddr8', FuncAddr, base_size=8, ins=(), outs=GPR, emit=''' PUT_OP(bits | (out_reg0 & 7), rex1(out_reg0), sink); sink.reloc_external(Reloc::Abs8, &func.dfg.ext_funcs[func_ref].name, 0); sink.put8(0); ''') # Similar to fnaddr4, but writes !0 (this is used by BaldrMonkey). allones_fnaddr4 = TailRecipe( 'allones_fnaddr4', FuncAddr, base_size=4, ins=(), outs=GPR, emit=''' PUT_OP(bits | (out_reg0 & 7), rex1(out_reg0), sink); sink.reloc_external(Reloc::Abs4, &func.dfg.ext_funcs[func_ref].name, 0); // Write the immediate as `!0` for the benefit of BaldrMonkey. sink.put4(!0); ''') # Similar to fnaddr8, but writes !0 (this is used by BaldrMonkey). allones_fnaddr8 = TailRecipe( 'allones_fnaddr8', FuncAddr, base_size=8, ins=(), outs=GPR, emit=''' PUT_OP(bits | (out_reg0 & 7), rex1(out_reg0), sink); sink.reloc_external(Reloc::Abs8, &func.dfg.ext_funcs[func_ref].name, 0); // Write the immediate as `!0` for the benefit of BaldrMonkey. sink.put8(!0); ''') pcrel_fnaddr8 = TailRecipe( 'pcrel_fnaddr8', FuncAddr, base_size=5, ins=(), outs=GPR, # rex2 gets passed 0 for r/m register because the upper bit of # r/m doesnt get decoded when in rip-relative addressing mode. emit=''' PUT_OP(bits, rex2(0, out_reg0), sink); modrm_riprel(out_reg0, sink); // The addend adjusts for the difference between the end of the // instruction and the beginning of the immediate field. sink.reloc_external(Reloc::X86PCRel4, &func.dfg.ext_funcs[func_ref].name, -4); sink.put4(0); ''') got_fnaddr8 = TailRecipe( 'got_fnaddr8', FuncAddr, base_size=5, ins=(), outs=GPR, # rex2 gets passed 0 for r/m register because the upper bit of # r/m doesnt get decoded when in rip-relative addressing mode. emit=''' PUT_OP(bits, rex2(0, out_reg0), sink); modrm_riprel(out_reg0, sink); // The addend adjusts for the difference between the end of the // instruction and the beginning of the immediate field. sink.reloc_external(Reloc::X86GOTPCRel4, &func.dfg.ext_funcs[func_ref].name, -4); sink.put4(0); ''') # XX+rd id with Abs4 globalsym relocation. gvaddr4 = TailRecipe( 'gvaddr4', UnaryGlobalValue, base_size=4, ins=(), outs=GPR, emit=''' PUT_OP(bits | (out_reg0 & 7), rex1(out_reg0), sink); sink.reloc_external(Reloc::Abs4, &func.global_values[global_value].symbol_name(), 0); sink.put4(0); ''') # XX+rd iq with Abs8 globalsym relocation. gvaddr8 = TailRecipe( 'gvaddr8', UnaryGlobalValue, base_size=8, ins=(), outs=GPR, emit=''' PUT_OP(bits | (out_reg0 & 7), rex1(out_reg0), sink); sink.reloc_external(Reloc::Abs8, &func.global_values[global_value].symbol_name(), 0); sink.put8(0); ''') # XX+rd iq with PCRel4 globalsym relocation. pcrel_gvaddr8 = TailRecipe( 'pcrel_gvaddr8', UnaryGlobalValue, base_size=5, ins=(), outs=GPR, emit=''' PUT_OP(bits, rex2(0, out_reg0), sink); modrm_rm(5, out_reg0, sink); // The addend adjusts for the difference between the end of the // instruction and the beginning of the immediate field. sink.reloc_external(Reloc::X86PCRel4, &func.global_values[global_value].symbol_name(), -4); sink.put4(0); ''') # XX+rd iq with Abs8 globalsym relocation. got_gvaddr8 = TailRecipe( 'got_gvaddr8', UnaryGlobalValue, base_size=5, ins=(), outs=GPR, emit=''' PUT_OP(bits, rex2(0, out_reg0), sink); modrm_rm(5, out_reg0, sink); // The addend adjusts for the difference between the end of the // instruction and the beginning of the immediate field. sink.reloc_external(Reloc::X86GOTPCRel4, &func.global_values[global_value].symbol_name(), -4); sink.put4(0); ''') # # Stack addresses. # # TODO: Alternative forms for 8-bit immediates, when applicable. # spaddr4_id = TailRecipe( 'spaddr4_id', StackLoad, base_size=6, ins=(), outs=GPR, emit=''' let sp = StackRef::sp(stack_slot, &func.stack_slots); let base = stk_base(sp.base); PUT_OP(bits, rex2(out_reg0, base), sink); modrm_sib_disp8(out_reg0, sink); sib_noindex(base, sink); let imm : i32 = offset.into(); sink.put4(sp.offset.checked_add(imm).unwrap() as u32); ''') spaddr8_id = TailRecipe( 'spaddr8_id', StackLoad, base_size=6, ins=(), outs=GPR, emit=''' let sp = StackRef::sp(stack_slot, &func.stack_slots); let base = stk_base(sp.base); PUT_OP(bits, rex2(base, out_reg0), sink); modrm_sib_disp32(out_reg0, sink); sib_noindex(base, sink); let imm : i32 = offset.into(); sink.put4(sp.offset.checked_add(imm).unwrap() as u32); ''') # # Store recipes. # # XX /r register-indirect store with no offset. st = TailRecipe( 'st', Store, base_size=1, ins=(GPR, GPR), outs=(), instp=IsEqual(Store.offset, 0), clobbers_flags=False, compute_size="size_plus_maybe_sib_or_offset_for_in_reg_1", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg1, in_reg0), sink); if needs_sib_byte(in_reg1) { modrm_sib(in_reg0, sink); sib_noindex(in_reg1, sink); } else if needs_offset(in_reg1) { modrm_disp8(in_reg1, in_reg0, sink); sink.put1(0); } else { modrm_rm(in_reg1, in_reg0, sink); } ''') # XX /r register-indirect store with index and no offset. stWithIndex = TailRecipe( 'stWithIndex', StoreComplex, base_size=2, ins=(GPR, GPR, GPR), outs=(), instp=IsEqual(StoreComplex.offset, 0), clobbers_flags=False, compute_size="size_plus_maybe_offset_for_in_reg_1", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink); // The else branch always inserts an SIB byte. if needs_offset(in_reg1) { modrm_sib_disp8(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); sink.put1(0); } else { modrm_sib(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); } ''') # XX /r register-indirect store with no offset. # Only ABCD allowed for stored value. This is for byte stores with no REX. st_abcd = TailRecipe( 'st_abcd', Store, base_size=1, ins=(ABCD, GPR), outs=(), instp=IsEqual(Store.offset, 0), when_prefixed=st, clobbers_flags=False, compute_size="size_plus_maybe_sib_or_offset_for_in_reg_1", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg1, in_reg0), sink); if needs_sib_byte(in_reg1) { modrm_sib(in_reg0, sink); sib_noindex(in_reg1, sink); } else if needs_offset(in_reg1) { modrm_disp8(in_reg1, in_reg0, sink); sink.put1(0); } else { modrm_rm(in_reg1, in_reg0, sink); } ''') # XX /r register-indirect store with index and no offset. # Only ABCD allowed for stored value. This is for byte stores with no REX. stWithIndex_abcd = TailRecipe( 'stWithIndex_abcd', StoreComplex, base_size=2, ins=(ABCD, GPR, GPR), outs=(), instp=IsEqual(StoreComplex.offset, 0), clobbers_flags=False, compute_size="size_plus_maybe_offset_for_in_reg_1", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink); // The else branch always inserts an SIB byte. if needs_offset(in_reg1) { modrm_sib_disp8(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); sink.put1(0); } else { modrm_sib(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); } ''') # XX /r register-indirect store of FPR with no offset. fst = TailRecipe( 'fst', Store, base_size=1, ins=(FPR, GPR), outs=(), instp=IsEqual(Store.offset, 0), clobbers_flags=False, compute_size="size_plus_maybe_sib_or_offset_for_in_reg_1", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg1, in_reg0), sink); if needs_sib_byte(in_reg1) { modrm_sib(in_reg0, sink); sib_noindex(in_reg1, sink); } else if needs_offset(in_reg1) { modrm_disp8(in_reg1, in_reg0, sink); sink.put1(0); } else { modrm_rm(in_reg1, in_reg0, sink); } ''') # XX /r register-indirect store with index and no offset of FPR. fstWithIndex = TailRecipe( 'fstWithIndex', StoreComplex, base_size=2, ins=(FPR, GPR, GPR), outs=(), instp=IsEqual(StoreComplex.offset, 0), clobbers_flags=False, compute_size="size_plus_maybe_offset_for_in_reg_1", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink); // The else branch always inserts an SIB byte. if needs_offset(in_reg1) { modrm_sib_disp8(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); sink.put1(0); } else { modrm_sib(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); } ''') # XX /r register-indirect store with 8-bit offset. stDisp8 = TailRecipe( 'stDisp8', Store, base_size=2, ins=(GPR, GPR), outs=(), instp=IsSignedInt(Store.offset, 8), clobbers_flags=False, compute_size="size_plus_maybe_sib_for_in_reg_1", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg1, in_reg0), sink); if needs_sib_byte(in_reg1) { modrm_sib_disp8(in_reg0, sink); sib_noindex(in_reg1, sink); } else { modrm_disp8(in_reg1, in_reg0, sink); } let offset: i32 = offset.into(); sink.put1(offset as u8); ''') # XX /r register-indirect store with index and 8-bit offset. stWithIndexDisp8 = TailRecipe( 'stWithIndexDisp8', StoreComplex, base_size=3, ins=(GPR, GPR, GPR), outs=(), instp=IsSignedInt(StoreComplex.offset, 8), clobbers_flags=False, emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink); modrm_sib_disp8(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); let offset: i32 = offset.into(); sink.put1(offset as u8); ''') # XX /r register-indirect store with 8-bit offset. # Only ABCD allowed for stored value. This is for byte stores with no REX. stDisp8_abcd = TailRecipe( 'stDisp8_abcd', Store, base_size=2, ins=(ABCD, GPR), outs=(), instp=IsSignedInt(Store.offset, 8), when_prefixed=stDisp8, clobbers_flags=False, compute_size="size_plus_maybe_sib_for_in_reg_1", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg1, in_reg0), sink); if needs_sib_byte(in_reg1) { modrm_sib_disp8(in_reg0, sink); sib_noindex(in_reg1, sink); } else { modrm_disp8(in_reg1, in_reg0, sink); } let offset: i32 = offset.into(); sink.put1(offset as u8); ''') # XX /r register-indirect store with index and 8-bit offset. # Only ABCD allowed for stored value. This is for byte stores with no REX. stWithIndexDisp8_abcd = TailRecipe( 'stWithIndexDisp8_abcd', StoreComplex, base_size=3, ins=(ABCD, GPR, GPR), outs=(), instp=IsSignedInt(StoreComplex.offset, 8), clobbers_flags=False, emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink); modrm_sib_disp8(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); let offset: i32 = offset.into(); sink.put1(offset as u8); ''') # XX /r register-indirect store with 8-bit offset of FPR. fstDisp8 = TailRecipe( 'fstDisp8', Store, base_size=2, ins=(FPR, GPR), outs=(), instp=IsSignedInt(Store.offset, 8), clobbers_flags=False, compute_size='size_plus_maybe_sib_for_in_reg_1', emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg1, in_reg0), sink); if needs_sib_byte(in_reg1) { modrm_sib_disp8(in_reg0, sink); sib_noindex(in_reg1, sink); } else { modrm_disp8(in_reg1, in_reg0, sink); } let offset: i32 = offset.into(); sink.put1(offset as u8); ''') # XX /r register-indirect store with index and 8-bit offset of FPR. fstWithIndexDisp8 = TailRecipe( 'fstWithIndexDisp8', StoreComplex, base_size=3, ins=(FPR, GPR, GPR), outs=(), instp=IsSignedInt(StoreComplex.offset, 8), clobbers_flags=False, emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink); modrm_sib_disp8(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); let offset: i32 = offset.into(); sink.put1(offset as u8); ''') # XX /r register-indirect store with 32-bit offset. stDisp32 = TailRecipe( 'stDisp32', Store, base_size=5, ins=(GPR, GPR), outs=(), clobbers_flags=False, compute_size='size_plus_maybe_sib_for_in_reg_1', emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg1, in_reg0), sink); if needs_sib_byte(in_reg1) { modrm_sib_disp32(in_reg0, sink); sib_noindex(in_reg1, sink); } else { modrm_disp32(in_reg1, in_reg0, sink); } let offset: i32 = offset.into(); sink.put4(offset as u32); ''') # XX /r register-indirect store with index and 32-bit offset. stWithIndexDisp32 = TailRecipe( 'stWithIndexDisp32', StoreComplex, base_size=6, ins=(GPR, GPR, GPR), outs=(), instp=IsSignedInt(StoreComplex.offset, 32), clobbers_flags=False, emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink); modrm_sib_disp32(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); let offset: i32 = offset.into(); sink.put4(offset as u32); ''') # XX /r register-indirect store with 32-bit offset. # Only ABCD allowed for stored value. This is for byte stores with no REX. stDisp32_abcd = TailRecipe( 'stDisp32_abcd', Store, base_size=5, ins=(ABCD, GPR), outs=(), when_prefixed=stDisp32, clobbers_flags=False, compute_size="size_plus_maybe_sib_for_in_reg_1", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg1, in_reg0), sink); if needs_sib_byte(in_reg1) { modrm_sib_disp32(in_reg0, sink); sib_noindex(in_reg1, sink); } else { modrm_disp32(in_reg1, in_reg0, sink); } let offset: i32 = offset.into(); sink.put4(offset as u32); ''') # XX /r register-indirect store with index and 32-bit offset. # Only ABCD allowed for stored value. This is for byte stores with no REX. stWithIndexDisp32_abcd = TailRecipe( 'stWithIndexDisp32_abcd', StoreComplex, base_size=6, ins=(ABCD, GPR, GPR), outs=(), instp=IsSignedInt(StoreComplex.offset, 32), clobbers_flags=False, emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink); modrm_sib_disp32(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); let offset: i32 = offset.into(); sink.put4(offset as u32); ''') # XX /r register-indirect store with 32-bit offset of FPR. fstDisp32 = TailRecipe( 'fstDisp32', Store, base_size=5, ins=(FPR, GPR), outs=(), clobbers_flags=False, compute_size='size_plus_maybe_sib_for_in_reg_1', emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg1, in_reg0), sink); if needs_sib_byte(in_reg1) { modrm_sib_disp32(in_reg0, sink); sib_noindex(in_reg1, sink); } else { modrm_disp32(in_reg1, in_reg0, sink); } let offset: i32 = offset.into(); sink.put4(offset as u32); ''') # XX /r register-indirect store with index and 32-bit offset of FPR. fstWithIndexDisp32 = TailRecipe( 'fstWithIndexDisp32', StoreComplex, base_size=6, ins=(FPR, GPR, GPR), outs=(), instp=IsSignedInt(StoreComplex.offset, 32), clobbers_flags=False, emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg1, in_reg0, in_reg2), sink); modrm_sib_disp32(in_reg0, sink); sib(0, in_reg2, in_reg1, sink); let offset: i32 = offset.into(); sink.put4(offset as u32); ''') # Unary spill with SIB and 32-bit displacement. spillSib32 = TailRecipe( 'spillSib32', Unary, base_size=6, ins=GPR, outs=StackGPR32, clobbers_flags=False, emit=''' sink.trap(TrapCode::StackOverflow, func.srclocs[inst]); let base = stk_base(out_stk0.base); PUT_OP(bits, rex2(base, in_reg0), sink); modrm_sib_disp32(in_reg0, sink); sib_noindex(base, sink); sink.put4(out_stk0.offset as u32); ''') # Like spillSib32, but targeting an FPR rather than a GPR. fspillSib32 = TailRecipe( 'fspillSib32', Unary, base_size=6, ins=FPR, outs=StackFPR32, clobbers_flags=False, emit=''' sink.trap(TrapCode::StackOverflow, func.srclocs[inst]); let base = stk_base(out_stk0.base); PUT_OP(bits, rex2(base, in_reg0), sink); modrm_sib_disp32(in_reg0, sink); sib_noindex(base, sink); sink.put4(out_stk0.offset as u32); ''') # Regspill using RSP-relative addressing. regspill32 = TailRecipe( 'regspill32', RegSpill, base_size=6, ins=GPR, outs=(), clobbers_flags=False, emit=''' sink.trap(TrapCode::StackOverflow, func.srclocs[inst]); let dst = StackRef::sp(dst, &func.stack_slots); let base = stk_base(dst.base); PUT_OP(bits, rex2(base, src), sink); modrm_sib_disp32(src, sink); sib_noindex(base, sink); sink.put4(dst.offset as u32); ''') # Like regspill32, but targeting an FPR rather than a GPR. fregspill32 = TailRecipe( 'fregspill32', RegSpill, base_size=6, ins=FPR, outs=(), clobbers_flags=False, emit=''' sink.trap(TrapCode::StackOverflow, func.srclocs[inst]); let dst = StackRef::sp(dst, &func.stack_slots); let base = stk_base(dst.base); PUT_OP(bits, rex2(base, src), sink); modrm_sib_disp32(src, sink); sib_noindex(base, sink); sink.put4(dst.offset as u32); ''') # # Load recipes # # XX /r load with no offset. ld = TailRecipe( 'ld', Load, base_size=1, ins=(GPR), outs=(GPR), instp=IsEqual(Load.offset, 0), clobbers_flags=False, compute_size="size_plus_maybe_sib_or_offset_for_in_reg_0", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg0, out_reg0), sink); if needs_sib_byte(in_reg0) { modrm_sib(out_reg0, sink); sib_noindex(in_reg0, sink); } else if needs_offset(in_reg0) { modrm_disp8(in_reg0, out_reg0, sink); sink.put1(0); } else { modrm_rm(in_reg0, out_reg0, sink); } ''') # XX /r load with index and no offset. ldWithIndex = TailRecipe( 'ldWithIndex', LoadComplex, base_size=2, ins=(GPR, GPR), outs=(GPR), instp=IsEqual(LoadComplex.offset, 0), clobbers_flags=False, compute_size="size_plus_maybe_offset_for_in_reg_0", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg0, out_reg0, in_reg1), sink); // The else branch always inserts an SIB byte. if needs_offset(in_reg0) { modrm_sib_disp8(out_reg0, sink); sib(0, in_reg1, in_reg0, sink); sink.put1(0); } else { modrm_sib(out_reg0, sink); sib(0, in_reg1, in_reg0, sink); } ''') # XX /r float load with no offset. fld = TailRecipe( 'fld', Load, base_size=1, ins=(GPR), outs=(FPR), instp=IsEqual(Load.offset, 0), clobbers_flags=False, compute_size="size_plus_maybe_sib_or_offset_for_in_reg_0", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg0, out_reg0), sink); if needs_sib_byte(in_reg0) { modrm_sib(out_reg0, sink); sib_noindex(in_reg0, sink); } else if needs_offset(in_reg0) { modrm_disp8(in_reg0, out_reg0, sink); sink.put1(0); } else { modrm_rm(in_reg0, out_reg0, sink); } ''') # XX /r float load with index and no offset. fldWithIndex = TailRecipe( 'fldWithIndex', LoadComplex, base_size=2, ins=(GPR, GPR), outs=(FPR), instp=IsEqual(LoadComplex.offset, 0), clobbers_flags=False, compute_size="size_plus_maybe_offset_for_in_reg_0", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg0, out_reg0, in_reg1), sink); // The else branch always inserts an SIB byte. if needs_offset(in_reg0) { modrm_sib_disp8(out_reg0, sink); sib(0, in_reg1, in_reg0, sink); sink.put1(0); } else { modrm_sib(out_reg0, sink); sib(0, in_reg1, in_reg0, sink); } ''') # XX /r load with 8-bit offset. ldDisp8 = TailRecipe( 'ldDisp8', Load, base_size=2, ins=(GPR), outs=(GPR), instp=IsSignedInt(Load.offset, 8), clobbers_flags=False, compute_size="size_plus_maybe_sib_for_in_reg_0", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg0, out_reg0), sink); if needs_sib_byte(in_reg0) { modrm_sib_disp8(out_reg0, sink); sib_noindex(in_reg0, sink); } else { modrm_disp8(in_reg0, out_reg0, sink); } let offset: i32 = offset.into(); sink.put1(offset as u8); ''') # XX /r load with index and 8-bit offset. ldWithIndexDisp8 = TailRecipe( 'ldWithIndexDisp8', LoadComplex, base_size=3, ins=(GPR, GPR), outs=(GPR), instp=IsSignedInt(LoadComplex.offset, 8), clobbers_flags=False, emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg0, out_reg0, in_reg1), sink); modrm_sib_disp8(out_reg0, sink); sib(0, in_reg1, in_reg0, sink); let offset: i32 = offset.into(); sink.put1(offset as u8); ''') # XX /r float load with 8-bit offset. fldDisp8 = TailRecipe( 'fldDisp8', Load, base_size=2, ins=(GPR), outs=(FPR), instp=IsSignedInt(Load.offset, 8), clobbers_flags=False, compute_size="size_plus_maybe_sib_for_in_reg_0", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg0, out_reg0), sink); if needs_sib_byte(in_reg0) { modrm_sib_disp8(out_reg0, sink); sib_noindex(in_reg0, sink); } else { modrm_disp8(in_reg0, out_reg0, sink); } let offset: i32 = offset.into(); sink.put1(offset as u8); ''') # XX /r float load with 8-bit offset. fldWithIndexDisp8 = TailRecipe( 'fldWithIndexDisp8', LoadComplex, base_size=3, ins=(GPR, GPR), outs=(FPR), instp=IsSignedInt(LoadComplex.offset, 8), clobbers_flags=False, emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg0, out_reg0, in_reg1), sink); modrm_sib_disp8(out_reg0, sink); sib(0, in_reg1, in_reg0, sink); let offset: i32 = offset.into(); sink.put1(offset as u8); ''') # XX /r load with 32-bit offset. ldDisp32 = TailRecipe( 'ldDisp32', Load, base_size=5, ins=(GPR), outs=(GPR), instp=IsSignedInt(Load.offset, 32), clobbers_flags=False, compute_size='size_plus_maybe_sib_for_in_reg_0', emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg0, out_reg0), sink); if needs_sib_byte(in_reg0) { modrm_sib_disp32(out_reg0, sink); sib_noindex(in_reg0, sink); } else { modrm_disp32(in_reg0, out_reg0, sink); } let offset: i32 = offset.into(); sink.put4(offset as u32); ''') # XX /r load with index and 32-bit offset. ldWithIndexDisp32 = TailRecipe( 'ldWithIndexDisp32', LoadComplex, base_size=6, ins=(GPR, GPR), outs=(GPR), instp=IsSignedInt(LoadComplex.offset, 32), clobbers_flags=False, emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg0, out_reg0, in_reg1), sink); modrm_sib_disp32(out_reg0, sink); sib(0, in_reg1, in_reg0, sink); let offset: i32 = offset.into(); sink.put4(offset as u32); ''') # XX /r float load with 32-bit offset. fldDisp32 = TailRecipe( 'fldDisp32', Load, base_size=5, ins=(GPR), outs=(FPR), instp=IsSignedInt(Load.offset, 32), clobbers_flags=False, compute_size="size_plus_maybe_sib_for_in_reg_0", emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex2(in_reg0, out_reg0), sink); if needs_sib_byte(in_reg0) { modrm_sib_disp32(out_reg0, sink); sib_noindex(in_reg0, sink); } else { modrm_disp32(in_reg0, out_reg0, sink); } let offset: i32 = offset.into(); sink.put4(offset as u32); ''') # XX /r float load with index and 32-bit offset. fldWithIndexDisp32 = TailRecipe( 'fldWithIndexDisp32', LoadComplex, base_size=6, ins=(GPR, GPR), outs=(FPR), instp=IsSignedInt(LoadComplex.offset, 32), clobbers_flags=False, emit=''' if !flags.notrap() { sink.trap(TrapCode::HeapOutOfBounds, func.srclocs[inst]); } PUT_OP(bits, rex3(in_reg0, out_reg0, in_reg1), sink); modrm_sib_disp32(out_reg0, sink); sib(0, in_reg1, in_reg0, sink); let offset: i32 = offset.into(); sink.put4(offset as u32); ''') # Unary fill with SIB and 32-bit displacement. fillSib32 = TailRecipe( 'fillSib32', Unary, base_size=6, ins=StackGPR32, outs=GPR, clobbers_flags=False, emit=''' let base = stk_base(in_stk0.base); PUT_OP(bits, rex2(base, out_reg0), sink); modrm_sib_disp32(out_reg0, sink); sib_noindex(base, sink); sink.put4(in_stk0.offset as u32); ''') # Like fillSib32, but targeting an FPR rather than a GPR. ffillSib32 = TailRecipe( 'ffillSib32', Unary, base_size=6, ins=StackFPR32, outs=FPR, clobbers_flags=False, emit=''' let base = stk_base(in_stk0.base); PUT_OP(bits, rex2(base, out_reg0), sink); modrm_sib_disp32(out_reg0, sink); sib_noindex(base, sink); sink.put4(in_stk0.offset as u32); ''') # Regfill with RSP-relative 32-bit displacement. regfill32 = TailRecipe( 'regfill32', RegFill, base_size=6, ins=StackGPR32, outs=(), clobbers_flags=False, emit=''' let src = StackRef::sp(src, &func.stack_slots); let base = stk_base(src.base); PUT_OP(bits, rex2(base, dst), sink); modrm_sib_disp32(dst, sink); sib_noindex(base, sink); sink.put4(src.offset as u32); ''') # Like regfill32, but targeting an FPR rather than a GPR. fregfill32 = TailRecipe( 'fregfill32', RegFill, base_size=6, ins=StackFPR32, outs=(), clobbers_flags=False, emit=''' let src = StackRef::sp(src, &func.stack_slots); let base = stk_base(src.base); PUT_OP(bits, rex2(base, dst), sink); modrm_sib_disp32(dst, sink); sib_noindex(base, sink); sink.put4(src.offset as u32); ''') # # Call/return # call_id = TailRecipe( 'call_id', Call, base_size=4, ins=(), outs=(), emit=''' sink.trap(TrapCode::StackOverflow, func.srclocs[inst]); PUT_OP(bits, BASE_REX, sink); // The addend adjusts for the difference between the end of the // instruction and the beginning of the immediate field. sink.reloc_external(Reloc::X86CallPCRel4, &func.dfg.ext_funcs[func_ref].name, -4); sink.put4(0); ''') call_plt_id = TailRecipe( 'call_plt_id', Call, base_size=4, ins=(), outs=(), emit=''' sink.trap(TrapCode::StackOverflow, func.srclocs[inst]); PUT_OP(bits, BASE_REX, sink); sink.reloc_external(Reloc::X86CallPLTRel4, &func.dfg.ext_funcs[func_ref].name, -4); sink.put4(0); ''') call_r = TailRecipe( 'call_r', CallIndirect, base_size=1, ins=GPR, outs=(), emit=''' sink.trap(TrapCode::StackOverflow, func.srclocs[inst]); PUT_OP(bits, rex1(in_reg0), sink); modrm_r_bits(in_reg0, bits, sink); ''') ret = TailRecipe( 'ret', MultiAry, base_size=0, ins=(), outs=(), emit=''' PUT_OP(bits, BASE_REX, sink); ''') # # Branches # jmpb = TailRecipe( 'jmpb', Jump, base_size=1, ins=(), outs=(), branch_range=8, clobbers_flags=False, emit=''' PUT_OP(bits, BASE_REX, sink); disp1(destination, func, sink); ''') jmpd = TailRecipe( 'jmpd', Jump, base_size=4, ins=(), outs=(), branch_range=32, clobbers_flags=False, emit=''' PUT_OP(bits, BASE_REX, sink); disp4(destination, func, sink); ''') brib = TailRecipe( 'brib', BranchInt, base_size=1, ins=FLAG.rflags, outs=(), branch_range=8, clobbers_flags=False, emit=''' PUT_OP(bits | icc2opc(cond), BASE_REX, sink); disp1(destination, func, sink); ''') brid = TailRecipe( 'brid', BranchInt, base_size=4, ins=FLAG.rflags, outs=(), branch_range=32, clobbers_flags=False, emit=''' PUT_OP(bits | icc2opc(cond), BASE_REX, sink); disp4(destination, func, sink); ''') brfb = TailRecipe( 'brfb', BranchFloat, base_size=1, ins=FLAG.rflags, outs=(), branch_range=8, clobbers_flags=False, instp=floatccs(BranchFloat), emit=''' PUT_OP(bits | fcc2opc(cond), BASE_REX, sink); disp1(destination, func, sink); ''') brfd = TailRecipe( 'brfd', BranchFloat, base_size=4, ins=FLAG.rflags, outs=(), branch_range=32, clobbers_flags=False, instp=floatccs(BranchFloat), emit=''' PUT_OP(bits | fcc2opc(cond), BASE_REX, sink); disp4(destination, func, sink); ''') indirect_jmp = TailRecipe( 'indirect_jmp', IndirectJump, base_size=1, ins=GPR, outs=(), clobbers_flags=False, emit=''' PUT_OP(bits, rex1(in_reg0), sink); modrm_r_bits(in_reg0, bits, sink); ''') jt_entry = TailRecipe( 'jt_entry', BranchTableEntry, base_size=2, ins=(GPR, GPR), outs=(GPR), clobbers_flags=False, instp=valid_scale(BranchTableEntry), compute_size="size_plus_maybe_offset_for_in_reg_1", emit=''' PUT_OP(bits, rex3(in_reg1, out_reg0, in_reg0), sink); if needs_offset(in_reg1) { modrm_sib_disp8(out_reg0, sink); sib(imm.trailing_zeros() as u8, in_reg0, in_reg1, sink); sink.put1(0); } else { modrm_sib(out_reg0, sink); sib(imm.trailing_zeros() as u8, in_reg0, in_reg1, sink); } ''') jt_base = TailRecipe( 'jt_base', BranchTableBase, base_size=5, ins=(), outs=(GPR), clobbers_flags=False, emit=''' PUT_OP(bits, rex2(0, out_reg0), sink); modrm_riprel(out_reg0, sink); // No reloc is needed here as the jump table is emitted directly after // the function body. jt_disp4(table, func, sink); ''') # # Test flags and set a register. # # These setCC instructions only set the low 8 bits, and they can only write # ABCD registers without a REX prefix. # # Other instruction encodings accepting `b1` inputs have the same constraints # and only look at the low 8 bits of the input register. # seti = TailRecipe( 'seti', IntCond, base_size=1, ins=FLAG.rflags, outs=GPR, requires_prefix=True, clobbers_flags=False, emit=''' PUT_OP(bits | icc2opc(cond), rex1(out_reg0), sink); modrm_r_bits(out_reg0, bits, sink); ''') seti_abcd = TailRecipe( 'seti_abcd', IntCond, base_size=1, ins=FLAG.rflags, outs=ABCD, when_prefixed=seti, clobbers_flags=False, emit=''' PUT_OP(bits | icc2opc(cond), rex1(out_reg0), sink); modrm_r_bits(out_reg0, bits, sink); ''') setf = TailRecipe( 'setf', FloatCond, base_size=1, ins=FLAG.rflags, outs=GPR, requires_prefix=True, clobbers_flags=False, emit=''' PUT_OP(bits | fcc2opc(cond), rex1(out_reg0), sink); modrm_r_bits(out_reg0, bits, sink); ''') setf_abcd = TailRecipe( 'setf_abcd', FloatCond, base_size=1, ins=FLAG.rflags, outs=ABCD, when_prefixed=setf, clobbers_flags=False, emit=''' PUT_OP(bits | fcc2opc(cond), rex1(out_reg0), sink); modrm_r_bits(out_reg0, bits, sink); ''') # # Conditional move (a.k.a integer select) # (maybe-REX.W) 0F 4x modrm(r,r) # 1 byte, modrm(r,r), is after the opcode # cmov = TailRecipe( 'cmov', IntSelect, base_size=1, ins=(FLAG.rflags, GPR, GPR), outs=2, requires_prefix=False, clobbers_flags=False, emit=''' PUT_OP(bits | icc2opc(cond), rex2(in_reg1, in_reg2), sink); modrm_rr(in_reg1, in_reg2, sink); ''') # # Bit scan forwards and reverse # bsf_and_bsr = TailRecipe( 'bsf_and_bsr', Unary, base_size=1, ins=GPR, outs=(GPR, FLAG.rflags), requires_prefix=False, clobbers_flags=True, emit=''' PUT_OP(bits, rex2(in_reg0, out_reg0), sink); modrm_rr(in_reg0, out_reg0, sink); ''') # # Compare and set flags. # # XX /r, MR form. Compare two GPR registers and set flags. rcmp = TailRecipe( 'rcmp', Binary, base_size=1, ins=(GPR, GPR), outs=FLAG.rflags, emit=''' PUT_OP(bits, rex2(in_reg0, in_reg1), sink); modrm_rr(in_reg0, in_reg1, sink); ''') # XX /r, RM form. Compare two FPR registers and set flags. fcmp = TailRecipe( 'fcmp', Binary, base_size=1, ins=(FPR, FPR), outs=FLAG.rflags, emit=''' PUT_OP(bits, rex2(in_reg1, in_reg0), sink); modrm_rr(in_reg1, in_reg0, sink); ''') # XX /n, MI form with imm8. rcmp_ib = TailRecipe( 'rcmp_ib', BinaryImm, base_size=2, ins=GPR, outs=FLAG.rflags, instp=IsSignedInt(BinaryImm.imm, 8), emit=''' PUT_OP(bits, rex1(in_reg0), sink); modrm_r_bits(in_reg0, bits, sink); let imm: i64 = imm.into(); sink.put1(imm as u8); ''') # XX /n, MI form with imm32. rcmp_id = TailRecipe( 'rcmp_id', BinaryImm, base_size=5, ins=GPR, outs=FLAG.rflags, instp=IsSignedInt(BinaryImm.imm, 32), emit=''' PUT_OP(bits, rex1(in_reg0), sink); modrm_r_bits(in_reg0, bits, sink); let imm: i64 = imm.into(); sink.put4(imm as u32); ''') # Same as rcmp, but second operand is the stack pointer. rcmp_sp = TailRecipe( 'rcmp_sp', Unary, base_size=1, ins=GPR, outs=FLAG.rflags, emit=''' PUT_OP(bits, rex2(in_reg0, RU::rsp.into()), sink); modrm_rr(in_reg0, RU::rsp.into(), sink); ''') # Test-and-branch. # # This recipe represents the macro fusion of a test and a conditional branch. # This serves two purposes: # # 1. Guarantee that the test and branch get scheduled next to each other so # macro fusion is guaranteed to be possible. # 2. Hide the status flags from Cranelift which doesn't currently model flags. # # The encoding bits affect both the test and the branch instruction: # # Bits 0-7 are the Jcc opcode. # Bits 8-15 control the test instruction which always has opcode byte 0x85. tjccb = TailRecipe( 'tjccb', Branch, base_size=1 + 2, ins=GPR, outs=(), branch_range=8, emit=''' // test r, r. PUT_OP((bits & 0xff00) | 0x85, rex2(in_reg0, in_reg0), sink); modrm_rr(in_reg0, in_reg0, sink); // Jcc instruction. sink.put1(bits as u8); disp1(destination, func, sink); ''') tjccd = TailRecipe( 'tjccd', Branch, base_size=1 + 6, ins=GPR, outs=(), branch_range=32, emit=''' // test r, r. PUT_OP((bits & 0xff00) | 0x85, rex2(in_reg0, in_reg0), sink); modrm_rr(in_reg0, in_reg0, sink); // Jcc instruction. sink.put1(0x0f); sink.put1(bits as u8); disp4(destination, func, sink); ''') # 8-bit test-and-branch. # # Same as tjccb, but only looks at the low 8 bits of the register, for b1 # types. t8jccb = TailRecipe( 't8jccb', Branch, base_size=1 + 2, ins=GPR, outs=(), branch_range=8, requires_prefix=True, emit=''' // test8 r, r. PUT_OP((bits & 0xff00) | 0x84, rex2(in_reg0, in_reg0), sink); modrm_rr(in_reg0, in_reg0, sink); // Jcc instruction. sink.put1(bits as u8); disp1(destination, func, sink); ''') t8jccb_abcd = TailRecipe( 't8jccb_abcd', Branch, base_size=1 + 2, ins=ABCD, outs=(), branch_range=8, when_prefixed=t8jccb, emit=''' // test8 r, r. PUT_OP((bits & 0xff00) | 0x84, rex2(in_reg0, in_reg0), sink); modrm_rr(in_reg0, in_reg0, sink); // Jcc instruction. sink.put1(bits as u8); disp1(destination, func, sink); ''') t8jccd = TailRecipe( 't8jccd', Branch, base_size=1 + 6, ins=GPR, outs=(), branch_range=32, requires_prefix=True, emit=''' // test8 r, r. PUT_OP((bits & 0xff00) | 0x84, rex2(in_reg0, in_reg0), sink); modrm_rr(in_reg0, in_reg0, sink); // Jcc instruction. sink.put1(0x0f); sink.put1(bits as u8); disp4(destination, func, sink); ''') t8jccd_abcd = TailRecipe( 't8jccd_abcd', Branch, base_size=1 + 6, ins=ABCD, outs=(), branch_range=32, when_prefixed=t8jccd, emit=''' // test8 r, r. PUT_OP((bits & 0xff00) | 0x84, rex2(in_reg0, in_reg0), sink); modrm_rr(in_reg0, in_reg0, sink); // Jcc instruction. sink.put1(0x0f); sink.put1(bits as u8); disp4(destination, func, sink); ''') # Worst case test-and-branch recipe for brz.b1 and brnz.b1 in 32-bit mode. # The register allocator can't handle a branch instruction with constrained # operands like the t8jccd_abcd above. This variant can accept the b1 opernd in # any register, but is is larger because it uses a 32-bit test instruction with # a 0xff immediate. t8jccd_long = TailRecipe( 't8jccd_long', Branch, base_size=5 + 6, ins=GPR, outs=(), branch_range=32, emit=''' // test32 r, 0xff. PUT_OP((bits & 0xff00) | 0xf7, rex1(in_reg0), sink); modrm_r_bits(in_reg0, bits, sink); sink.put4(0xff); // Jcc instruction. sink.put1(0x0f); sink.put1(bits as u8); disp4(destination, func, sink); ''') # Comparison that produces a `b1` result in a GPR. # # This is a macro of a `cmp` instruction followed by a `setCC` instruction. # This is not a great solution because: # # - The cmp+setcc combination is not recognized by CPU's macro fusion. # - The 64-bit encoding has issues with REX prefixes. The `cmp` and `setCC` # instructions may need a REX independently. # - Modeling CPU flags in the type system would be better. # # Since the `setCC` instructions only write an 8-bit register, we use that as # our `b1` representation: A `b1` value is represented as a GPR where the low 8 # bits are known to be 0 or 1. The high bits are undefined. # # This bandaid macro doesn't support a REX prefix for the final `setCC` # instruction, so it is limited to the `ABCD` register class for booleans. # The omission of a `when_prefixed` alternative is deliberate here. icscc = TailRecipe( 'icscc', IntCompare, base_size=1 + 3, ins=(GPR, GPR), outs=ABCD, emit=''' // Comparison instruction. PUT_OP(bits, rex2(in_reg0, in_reg1), sink); modrm_rr(in_reg0, in_reg1, sink); // `setCC` instruction, no REX. use crate::ir::condcodes::IntCC::*; let setcc = match cond { Equal => 0x94, NotEqual => 0x95, SignedLessThan => 0x9c, SignedGreaterThanOrEqual => 0x9d, SignedGreaterThan => 0x9f, SignedLessThanOrEqual => 0x9e, UnsignedLessThan => 0x92, UnsignedGreaterThanOrEqual => 0x93, UnsignedGreaterThan => 0x97, UnsignedLessThanOrEqual => 0x96, }; sink.put1(0x0f); sink.put1(setcc); modrm_rr(out_reg0, 0, sink); ''') icscc_ib = TailRecipe( 'icscc_ib', IntCompareImm, base_size=2 + 3, ins=GPR, outs=ABCD, instp=IsSignedInt(IntCompareImm.imm, 8), emit=''' // Comparison instruction. PUT_OP(bits, rex1(in_reg0), sink); modrm_r_bits(in_reg0, bits, sink); let imm: i64 = imm.into(); sink.put1(imm as u8); // `setCC` instruction, no REX. use crate::ir::condcodes::IntCC::*; let setcc = match cond { Equal => 0x94, NotEqual => 0x95, SignedLessThan => 0x9c, SignedGreaterThanOrEqual => 0x9d, SignedGreaterThan => 0x9f, SignedLessThanOrEqual => 0x9e, UnsignedLessThan => 0x92, UnsignedGreaterThanOrEqual => 0x93, UnsignedGreaterThan => 0x97, UnsignedLessThanOrEqual => 0x96, }; sink.put1(0x0f); sink.put1(setcc); modrm_rr(out_reg0, 0, sink); ''') icscc_id = TailRecipe( 'icscc_id', IntCompareImm, base_size=5 + 3, ins=GPR, outs=ABCD, instp=IsSignedInt(IntCompareImm.imm, 32), emit=''' // Comparison instruction. PUT_OP(bits, rex1(in_reg0), sink); modrm_r_bits(in_reg0, bits, sink); let imm: i64 = imm.into(); sink.put4(imm as u32); // `setCC` instruction, no REX. use crate::ir::condcodes::IntCC::*; let setcc = match cond { Equal => 0x94, NotEqual => 0x95, SignedLessThan => 0x9c, SignedGreaterThanOrEqual => 0x9d, SignedGreaterThan => 0x9f, SignedLessThanOrEqual => 0x9e, UnsignedLessThan => 0x92, UnsignedGreaterThanOrEqual => 0x93, UnsignedGreaterThan => 0x97, UnsignedLessThanOrEqual => 0x96, }; sink.put1(0x0f); sink.put1(setcc); modrm_rr(out_reg0, 0, sink); ''') # Make a FloatCompare instruction predicate with the supported condition codes. # Same thing for floating point. # # The ucomiss/ucomisd instructions set the FLAGS bits CF/PF/CF like this: # # ZPC OSA # UN 111 000 # GT 000 000 # LT 001 000 # EQ 100 000 # # Not all floating point condition codes are supported. # The omission of a `when_prefixed` alternative is deliberate here. fcscc = TailRecipe( 'fcscc', FloatCompare, base_size=1 + 3, ins=(FPR, FPR), outs=ABCD, instp=floatccs(FloatCompare), emit=''' // Comparison instruction. PUT_OP(bits, rex2(in_reg1, in_reg0), sink); modrm_rr(in_reg1, in_reg0, sink); // `setCC` instruction, no REX. use crate::ir::condcodes::FloatCC::*; let setcc = match cond { Ordered => 0x9b, // EQ|LT|GT => setnp (P=0) Unordered => 0x9a, // UN => setp (P=1) OrderedNotEqual => 0x95, // LT|GT => setne (Z=0), UnorderedOrEqual => 0x94, // UN|EQ => sete (Z=1) GreaterThan => 0x97, // GT => seta (C=0&Z=0) GreaterThanOrEqual => 0x93, // GT|EQ => setae (C=0) UnorderedOrLessThan => 0x92, // UN|LT => setb (C=1) UnorderedOrLessThanOrEqual => 0x96, // UN|LT|EQ => setbe (Z=1|C=1) Equal | // EQ NotEqual | // UN|LT|GT LessThan | // LT LessThanOrEqual | // LT|EQ UnorderedOrGreaterThan | // UN|GT UnorderedOrGreaterThanOrEqual // UN|GT|EQ => panic!("{} not supported by fcscc", cond), }; sink.put1(0x0f); sink.put1(setcc); modrm_rr(out_reg0, 0, sink); ''') TailRecipe.check_names(globals())